[ 531.495078] env[67752]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'linux_bridge' {{(pid=67752) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 531.495427] env[67752]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'noop' {{(pid=67752) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 531.495529] env[67752]: DEBUG os_vif [-] Loaded VIF plugin class '' with name 'ovs' {{(pid=67752) initialize /opt/stack/data/venv/lib/python3.10/site-packages/os_vif/__init__.py:44}} [ 531.495862] env[67752]: INFO os_vif [-] Loaded VIF plugins: linux_bridge, noop, ovs [ 531.581036] env[67752]: DEBUG oslo_concurrency.processutils [-] Running cmd (subprocess): grep -F node.session.scan /sbin/iscsiadm {{(pid=67752) execute /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/processutils.py:390}} [ 531.591688] env[67752]: DEBUG oslo_concurrency.processutils [-] CMD "grep -F node.session.scan /sbin/iscsiadm" returned: 0 in 0.011s {{(pid=67752) execute /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/processutils.py:428}} [ 531.729613] env[67752]: INFO nova.virt.driver [None req-4a2a4595-576b-42f0-91c4-4f3bfb4fd377 None None] Loading compute driver 'vmwareapi.VMwareVCDriver' [ 531.801015] env[67752]: DEBUG oslo_concurrency.lockutils [-] Acquiring lock "oslo_vmware_api_lock" by "oslo_vmware.api.VMwareAPISession._create_session" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 531.801220] env[67752]: DEBUG oslo_concurrency.lockutils [-] Lock "oslo_vmware_api_lock" acquired by "oslo_vmware.api.VMwareAPISession._create_session" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 531.801311] env[67752]: DEBUG oslo_vmware.service [-] Creating suds client with soap_url='https://vc1.osci.c.eu-de-1.cloud.sap:443/sdk' and wsdl_url='https://vc1.osci.c.eu-de-1.cloud.sap:443/sdk/vimService.wsdl' {{(pid=67752) __init__ /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:242}} [ 534.958021] env[67752]: DEBUG oslo_vmware.service [-] Invoking ServiceInstance.RetrieveServiceContent with opID=oslo.vmware-d01e6981-ae54-474b-93ec-d59ab90991c3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.974293] env[67752]: DEBUG oslo_vmware.api [-] Logging into host: vc1.osci.c.eu-de-1.cloud.sap. {{(pid=67752) _create_session /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:242}} [ 534.974436] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.Login with opID=oslo.vmware-0622cdef-abc4-4fd6-9a7c-ec72026e78bf {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 534.999544] env[67752]: INFO oslo_vmware.api [-] Successfully established new session; session ID is 05875. [ 534.999659] env[67752]: DEBUG oslo_concurrency.lockutils [-] Lock "oslo_vmware_api_lock" "released" by "oslo_vmware.api.VMwareAPISession._create_session" :: held 3.199s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 535.000342] env[67752]: INFO nova.virt.vmwareapi.driver [None req-4a2a4595-576b-42f0-91c4-4f3bfb4fd377 None None] VMware vCenter version: 7.0.3 [ 535.003740] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-740607ad-0f1b-4545-8fff-c72f76dfdec0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 535.024536] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db594b56-4584-42b3-ac73-619d7a1cdf29 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 535.030237] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-18b0f1af-eeb0-4075-aa9d-ee95d6a3c0be {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 535.036638] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e815229-c14e-4abc-b13d-0460522e41b9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 535.049222] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1182d596-e759-4fca-b7c2-b741dd803dfa {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 535.055013] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-be0a7d4a-c007-4d84-b523-71b21274a28d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 535.084366] env[67752]: DEBUG oslo_vmware.service [-] Invoking ExtensionManager.FindExtension with opID=oslo.vmware-75e079eb-50ef-4479-a82c-33a12892111e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 535.089119] env[67752]: DEBUG nova.virt.vmwareapi.driver [None req-4a2a4595-576b-42f0-91c4-4f3bfb4fd377 None None] Extension org.openstack.compute already exists. {{(pid=67752) _register_openstack_extension /opt/stack/nova/nova/virt/vmwareapi/driver.py:224}} [ 535.091785] env[67752]: INFO nova.compute.provider_config [None req-4a2a4595-576b-42f0-91c4-4f3bfb4fd377 None None] No provider configs found in /etc/nova/provider_config/. If files are present, ensure the Nova process has access. [ 535.112993] env[67752]: DEBUG nova.context [None req-4a2a4595-576b-42f0-91c4-4f3bfb4fd377 None None] Found 2 cells: 00000000-0000-0000-0000-000000000000(cell0),17abf971-7250-4e4f-919b-ddf17ef1f413(cell1) {{(pid=67752) load_cells /opt/stack/nova/nova/context.py:464}} [ 535.114911] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.115141] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.115836] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 535.116270] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] Acquiring lock "17abf971-7250-4e4f-919b-ddf17ef1f413" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.116463] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] Lock "17abf971-7250-4e4f-919b-ddf17ef1f413" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.117476] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] Lock "17abf971-7250-4e4f-919b-ddf17ef1f413" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 535.137788] env[67752]: INFO dbcounter [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] Registered counter for database nova_cell0 [ 535.146288] env[67752]: INFO dbcounter [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] Registered counter for database nova_cell1 [ 535.149463] env[67752]: DEBUG oslo_db.sqlalchemy.engines [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=67752) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 535.149812] env[67752]: DEBUG oslo_db.sqlalchemy.engines [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=67752) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 535.154373] env[67752]: DEBUG dbcounter [-] [67752] Writer thread running {{(pid=67752) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 535.155453] env[67752]: DEBUG dbcounter [-] [67752] Writer thread running {{(pid=67752) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 535.157295] env[67752]: ERROR nova.db.main.api [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] No DB access allowed in nova-compute: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main [ 535.157295] env[67752]: result = function(*args, **kwargs) [ 535.157295] env[67752]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 535.157295] env[67752]: return func(*args, **kwargs) [ 535.157295] env[67752]: File "/opt/stack/nova/nova/context.py", line 422, in gather_result [ 535.157295] env[67752]: result = fn(*args, **kwargs) [ 535.157295] env[67752]: File "/opt/stack/nova/nova/db/main/api.py", line 179, in wrapper [ 535.157295] env[67752]: return f(*args, **kwargs) [ 535.157295] env[67752]: File "/opt/stack/nova/nova/objects/service.py", line 553, in _db_service_get_minimum_version [ 535.157295] env[67752]: return db.service_get_minimum_version(context, binaries) [ 535.157295] env[67752]: File "/opt/stack/nova/nova/db/main/api.py", line 238, in wrapper [ 535.157295] env[67752]: _check_db_access() [ 535.157295] env[67752]: File "/opt/stack/nova/nova/db/main/api.py", line 188, in _check_db_access [ 535.157295] env[67752]: stacktrace = ''.join(traceback.format_stack()) [ 535.157295] env[67752]: [ 535.158325] env[67752]: ERROR nova.db.main.api [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] No DB access allowed in nova-compute: File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main [ 535.158325] env[67752]: result = function(*args, **kwargs) [ 535.158325] env[67752]: File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper [ 535.158325] env[67752]: return func(*args, **kwargs) [ 535.158325] env[67752]: File "/opt/stack/nova/nova/context.py", line 422, in gather_result [ 535.158325] env[67752]: result = fn(*args, **kwargs) [ 535.158325] env[67752]: File "/opt/stack/nova/nova/db/main/api.py", line 179, in wrapper [ 535.158325] env[67752]: return f(*args, **kwargs) [ 535.158325] env[67752]: File "/opt/stack/nova/nova/objects/service.py", line 553, in _db_service_get_minimum_version [ 535.158325] env[67752]: return db.service_get_minimum_version(context, binaries) [ 535.158325] env[67752]: File "/opt/stack/nova/nova/db/main/api.py", line 238, in wrapper [ 535.158325] env[67752]: _check_db_access() [ 535.158325] env[67752]: File "/opt/stack/nova/nova/db/main/api.py", line 188, in _check_db_access [ 535.158325] env[67752]: stacktrace = ''.join(traceback.format_stack()) [ 535.158325] env[67752]: [ 535.158745] env[67752]: WARNING nova.objects.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] Failed to get minimum service version for cell 17abf971-7250-4e4f-919b-ddf17ef1f413 [ 535.158816] env[67752]: WARNING nova.objects.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] Failed to get minimum service version for cell 00000000-0000-0000-0000-000000000000 [ 535.159248] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] Acquiring lock "singleton_lock" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 535.159416] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] Acquired lock "singleton_lock" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 535.159664] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] Releasing lock "singleton_lock" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 535.160030] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] Full set of CONF: {{(pid=67752) _wait_for_exit_or_signal /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/service.py:363}} [ 535.160203] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ******************************************************************************** {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2600}} [ 535.160349] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] Configuration options gathered from: {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2601}} [ 535.160499] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] command line args: ['--config-file', '/etc/nova/nova.conf', '--config-file', '/etc/nova/nova-cpu-common.conf', '--config-file', '/etc/nova/nova-cpu-1.conf'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2602}} [ 535.160692] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] config files: ['/etc/nova/nova.conf', '/etc/nova/nova-cpu-common.conf', '/etc/nova/nova-cpu-1.conf'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2603}} [ 535.160824] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ================================================================================ {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2605}} [ 535.161051] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] allow_resize_to_same_host = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.161224] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] arq_binding_timeout = 300 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.161360] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] backdoor_port = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.161490] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] backdoor_socket = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.161655] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] block_device_allocate_retries = 60 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.161878] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] block_device_allocate_retries_interval = 3 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.161985] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cert = self.pem {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.162167] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] compute_driver = vmwareapi.VMwareVCDriver {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.162338] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] compute_monitors = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.162505] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] config_dir = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.162674] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] config_drive_format = iso9660 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.162814] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] config_file = ['/etc/nova/nova.conf', '/etc/nova/nova-cpu-common.conf', '/etc/nova/nova-cpu-1.conf'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.162979] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] config_source = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.163164] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] console_host = devstack {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.163369] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] control_exchange = nova {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.163546] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cpu_allocation_ratio = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.163710] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] daemon = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.163876] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] debug = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.164049] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] default_access_ip_network_name = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.164223] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] default_availability_zone = nova {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.164381] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] default_ephemeral_format = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.164539] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] default_green_pool_size = 1000 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.164768] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] default_log_levels = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN', 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO', 'oslo.messaging=INFO', 'oslo_messaging=INFO', 'iso8601=WARN', 'requests.packages.urllib3.connectionpool=WARN', 'urllib3.connectionpool=WARN', 'websocket=WARN', 'requests.packages.urllib3.util.retry=WARN', 'urllib3.util.retry=WARN', 'keystonemiddleware=WARN', 'routes.middleware=WARN', 'stevedore=WARN', 'taskflow=WARN', 'keystoneauth=WARN', 'oslo.cache=INFO', 'oslo_policy=INFO', 'dogpile.core.dogpile=INFO', 'glanceclient=WARN', 'oslo.privsep.daemon=INFO'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.164935] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] default_schedule_zone = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.165107] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] disk_allocation_ratio = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.165272] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] enable_new_services = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.165451] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] enabled_apis = ['osapi_compute'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.165612] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] enabled_ssl_apis = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.165774] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] flat_injected = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.165934] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] force_config_drive = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.166109] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] force_raw_images = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.166297] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] graceful_shutdown_timeout = 5 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.166475] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] heal_instance_info_cache_interval = 60 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.166695] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] host = cpu-1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.166872] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] initial_cpu_allocation_ratio = 4.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.167047] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] initial_disk_allocation_ratio = 1.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.167214] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] initial_ram_allocation_ratio = 1.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.167460] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] injected_network_template = /opt/stack/nova/nova/virt/interfaces.template {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.167630] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] instance_build_timeout = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.167793] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] instance_delete_interval = 300 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.167963] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] instance_format = [instance: %(uuid)s] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.168151] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] instance_name_template = instance-%08x {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.168317] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] instance_usage_audit = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.168491] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] instance_usage_audit_period = month {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.168658] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] instance_uuid_format = [instance: %(uuid)s] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.168823] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] instances_path = /opt/stack/data/nova/instances {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.168988] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] internal_service_availability_zone = internal {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.169162] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] key = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.169346] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] live_migration_retry_count = 30 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.169529] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] log_config_append = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.169701] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] log_date_format = %Y-%m-%d %H:%M:%S {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.169862] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] log_dir = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.170046] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] log_file = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.170186] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] log_options = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.170360] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] log_rotate_interval = 1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.170536] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] log_rotate_interval_type = days {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.170704] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] log_rotation_type = none {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.170838] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] logging_context_format_string = %(color)s%(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(project_name)s %(user_name)s%(color)s] %(instance)s%(color)s%(message)s {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.170967] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] logging_debug_format_suffix = {{(pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d}} {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.171171] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] logging_default_format_string = %(color)s%(levelname)s %(name)s [-%(color)s] %(instance)s%(color)s%(message)s {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.171363] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] logging_exception_prefix = ERROR %(name)s %(instance)s {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.171497] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.171664] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] long_rpc_timeout = 1800 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.171826] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] max_concurrent_builds = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.171985] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] max_concurrent_live_migrations = 1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.172160] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] max_concurrent_snapshots = 5 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.172330] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] max_local_block_devices = 3 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.172508] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] max_logfile_count = 30 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.172674] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] max_logfile_size_mb = 200 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.172836] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] maximum_instance_delete_attempts = 5 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.173015] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] metadata_listen = 0.0.0.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.173189] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] metadata_listen_port = 8775 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.173363] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] metadata_workers = 2 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.173522] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] migrate_max_retries = -1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.173725] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] mkisofs_cmd = genisoimage {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.173943] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] my_block_storage_ip = 10.180.1.21 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.174090] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] my_ip = 10.180.1.21 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.174260] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] network_allocate_retries = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.174443] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] non_inheritable_image_properties = ['cache_in_nova', 'bittorrent'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.174614] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] osapi_compute_listen = 0.0.0.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.174774] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] osapi_compute_listen_port = 8774 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.174941] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] osapi_compute_unique_server_name_scope = {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.175122] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] osapi_compute_workers = 2 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.175292] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] password_length = 12 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.175493] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] periodic_enable = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.175663] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] periodic_fuzzy_delay = 60 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.175836] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] pointer_model = usbtablet {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.176024] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] preallocate_images = none {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.176186] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] publish_errors = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.176320] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] pybasedir = /opt/stack/nova {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.176480] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ram_allocation_ratio = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.176638] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] rate_limit_burst = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.176804] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] rate_limit_except_level = CRITICAL {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.176964] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] rate_limit_interval = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.177138] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] reboot_timeout = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.177300] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] reclaim_instance_interval = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.177460] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] record = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.177630] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] reimage_timeout_per_gb = 60 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.177796] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] report_interval = 120 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.177955] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] rescue_timeout = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.178130] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] reserved_host_cpus = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.178316] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] reserved_host_disk_mb = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.178479] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] reserved_host_memory_mb = 512 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.178640] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] reserved_huge_pages = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.178799] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] resize_confirm_window = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.178958] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] resize_fs_using_block_device = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.179130] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] resume_guests_state_on_host_boot = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.179322] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] rootwrap_config = /etc/nova/rootwrap.conf {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.179539] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] rpc_response_timeout = 60 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.179714] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] run_external_periodic_tasks = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.179890] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] running_deleted_instance_action = reap {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.180067] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] running_deleted_instance_poll_interval = 1800 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.180260] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] running_deleted_instance_timeout = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.180433] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] scheduler_instance_sync_interval = 120 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.180603] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] service_down_time = 720 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.180776] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] servicegroup_driver = db {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.180938] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] shelved_offload_time = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.181112] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] shelved_poll_interval = 3600 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.181308] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] shutdown_timeout = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.181477] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] source_is_ipv6 = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.181638] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ssl_only = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.181888] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] state_path = /opt/stack/data/n-cpu-1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.182070] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] sync_power_state_interval = 600 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.182237] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] sync_power_state_pool_size = 1000 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.182409] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] syslog_log_facility = LOG_USER {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.182568] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] tempdir = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.182729] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] timeout_nbd = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.182897] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] transport_url = **** {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.183069] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] update_resources_interval = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.183254] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] use_cow_images = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.183433] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] use_eventlog = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.183594] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] use_journal = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.183754] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] use_json = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.183911] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] use_rootwrap_daemon = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.184079] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] use_stderr = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.184262] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] use_syslog = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.184438] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vcpu_pin_set = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.184608] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vif_plugging_is_fatal = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.184775] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vif_plugging_timeout = 300 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.184941] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] virt_mkfs = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.185116] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] volume_usage_poll_interval = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.185279] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] watch_log_file = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.185450] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] web = /usr/share/spice-html5 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 535.185633] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_concurrency.disable_process_locking = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.185921] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_concurrency.lock_path = /opt/stack/data/n-cpu-1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.186117] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_metrics.metrics_buffer_size = 1000 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.186290] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_metrics.metrics_enabled = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.186490] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_metrics.metrics_process_name = {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.186663] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_metrics.metrics_socket_file = /var/tmp/metrics_collector.sock {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.186834] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_metrics.metrics_thread_stop_timeout = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.187025] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.auth_strategy = keystone {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.187199] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.compute_link_prefix = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.187416] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.config_drive_skip_versions = 1.0 2007-01-19 2007-03-01 2007-08-29 2007-10-10 2007-12-15 2008-02-01 2008-09-01 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.187599] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.dhcp_domain = novalocal {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.187771] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.enable_instance_password = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.187939] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.glance_link_prefix = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.188120] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.instance_list_cells_batch_fixed_size = 100 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.188300] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.instance_list_cells_batch_strategy = distributed {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.188468] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.instance_list_per_project_cells = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.188634] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.list_records_by_skipping_down_cells = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.188800] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.local_metadata_per_cell = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.188998] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.max_limit = 1000 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.189193] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.metadata_cache_expiration = 15 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.189375] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.neutron_default_tenant_id = default {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.189549] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.use_neutron_default_nets = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.189718] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.vendordata_dynamic_connect_timeout = 5 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.189886] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.vendordata_dynamic_failure_fatal = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.190075] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.vendordata_dynamic_read_timeout = 5 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.190290] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.vendordata_dynamic_ssl_certfile = {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.190483] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.vendordata_dynamic_targets = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.190658] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.vendordata_jsonfile_path = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.190842] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api.vendordata_providers = ['StaticJSON'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.191049] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.backend = dogpile.cache.memcached {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.191256] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.backend_argument = **** {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.191443] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.config_prefix = cache.oslo {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.191617] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.dead_timeout = 60.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.191787] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.debug_cache_backend = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.191953] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.enable_retry_client = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.192134] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.enable_socket_keepalive = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.192310] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.enabled = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.192480] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.enforce_fips_mode = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.192646] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.expiration_time = 600 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.192812] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.hashclient_retry_attempts = 2 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.192980] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.hashclient_retry_delay = 1.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.193162] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.memcache_dead_retry = 300 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.193350] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.memcache_password = **** {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.193531] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.memcache_pool_connection_get_timeout = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.193700] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.memcache_pool_flush_on_reconnect = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.193864] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.memcache_pool_maxsize = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.194040] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.memcache_pool_unused_timeout = 60 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.194209] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.memcache_sasl_enabled = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.194394] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.memcache_servers = ['localhost:11211'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.194561] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.memcache_socket_timeout = 1.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.194725] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.memcache_username = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.194891] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.proxies = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.195065] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.redis_password = **** {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.195255] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.redis_sentinel_service_name = mymaster {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.195453] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.redis_sentinels = ['localhost:26379'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.195633] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.redis_server = localhost:6379 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.195803] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.redis_socket_timeout = 1.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.195966] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.redis_username = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.196149] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.retry_attempts = 2 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.196335] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.retry_delay = 0.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.196519] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.socket_keepalive_count = 1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.196687] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.socket_keepalive_idle = 1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.196851] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.socket_keepalive_interval = 1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.197022] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.tls_allowed_ciphers = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.197189] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.tls_cafile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.197350] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.tls_certfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.197513] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.tls_enabled = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.197672] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cache.tls_keyfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.197846] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cinder.auth_section = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.198030] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cinder.auth_type = password {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.198199] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cinder.cafile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.198382] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cinder.catalog_info = volumev3::publicURL {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.198546] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cinder.certfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.198712] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cinder.collect_timing = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.198878] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cinder.cross_az_attach = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.199083] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cinder.debug = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.199264] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cinder.endpoint_template = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.199469] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cinder.http_retries = 3 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.199643] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cinder.insecure = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.199808] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cinder.keyfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.199984] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cinder.os_region_name = RegionOne {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.200181] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cinder.split_loggers = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.200361] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cinder.timeout = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.200540] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] compute.consecutive_build_service_disable_threshold = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.200706] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] compute.cpu_dedicated_set = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.200870] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] compute.cpu_shared_set = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.201056] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] compute.image_type_exclude_list = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.201247] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] compute.live_migration_wait_for_vif_plug = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.201430] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] compute.max_concurrent_disk_ops = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.201590] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] compute.max_disk_devices_to_attach = -1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.201756] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] compute.packing_host_numa_cells_allocation_strategy = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.201927] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] compute.provider_config_location = /etc/nova/provider_config/ {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.202109] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] compute.resource_provider_association_refresh = 300 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.202279] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] compute.shutdown_retry_interval = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.202495] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] compute.vmdk_allowed_types = ['streamOptimized', 'monolithicSparse'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.202678] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] conductor.workers = 2 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.202859] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] console.allowed_origins = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.203035] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] console.ssl_ciphers = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.203230] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] console.ssl_minimum_version = default {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.203424] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] consoleauth.enforce_session_timeout = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.203598] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] consoleauth.token_ttl = 600 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.203768] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.cafile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.203932] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.certfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.204110] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.collect_timing = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.204275] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.connect_retries = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.204437] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.connect_retry_delay = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.204595] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.endpoint_override = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.204759] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.insecure = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.204918] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.keyfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.205089] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.max_version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.205251] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.min_version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.205432] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.region_name = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.205634] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.retriable_status_codes = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.205803] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.service_name = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.205978] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.service_type = accelerator {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.206158] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.split_loggers = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.206324] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.status_code_retries = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.206484] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.status_code_retry_delay = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.206646] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.timeout = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.206826] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.valid_interfaces = ['internal', 'public'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.206990] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] cyborg.version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.207189] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.backend = sqlalchemy {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.207366] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.connection = **** {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.207622] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.connection_debug = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.207748] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.connection_parameters = {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.207955] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.connection_recycle_time = 3600 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.208177] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.connection_trace = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.208312] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.db_inc_retry_interval = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.208483] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.db_max_retries = 20 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.208642] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.db_max_retry_interval = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.208806] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.db_retry_interval = 1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.208969] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.max_overflow = 50 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.209153] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.max_pool_size = 5 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.210021] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.max_retries = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.210021] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.mysql_sql_mode = TRADITIONAL {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.210021] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.mysql_wsrep_sync_wait = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.210021] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.pool_timeout = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.210021] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.retry_interval = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.210203] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.slave_connection = **** {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.210286] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.sqlite_synchronous = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.210465] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] database.use_db_reconnect = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.210649] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api_database.backend = sqlalchemy {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.210855] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api_database.connection = **** {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.211052] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api_database.connection_debug = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.211253] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api_database.connection_parameters = {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.211429] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api_database.connection_recycle_time = 3600 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.211642] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api_database.connection_trace = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.211758] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api_database.db_inc_retry_interval = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.211920] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api_database.db_max_retries = 20 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.212095] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api_database.db_max_retry_interval = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.212264] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api_database.db_retry_interval = 1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.212429] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api_database.max_overflow = 50 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.212591] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api_database.max_pool_size = 5 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.212754] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api_database.max_retries = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.212925] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api_database.mysql_sql_mode = TRADITIONAL {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.213097] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api_database.mysql_wsrep_sync_wait = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.213300] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api_database.pool_timeout = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.213487] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api_database.retry_interval = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.213687] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api_database.slave_connection = **** {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.213899] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] api_database.sqlite_synchronous = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.214117] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] devices.enabled_mdev_types = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.214310] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ephemeral_storage_encryption.cipher = aes-xts-plain64 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.214491] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ephemeral_storage_encryption.default_format = luks {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.214659] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ephemeral_storage_encryption.enabled = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.214827] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ephemeral_storage_encryption.key_size = 512 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.215008] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.api_servers = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.215191] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.cafile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.215392] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.certfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.215565] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.collect_timing = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.215730] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.connect_retries = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.215891] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.connect_retry_delay = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.216067] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.debug = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.216238] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.default_trusted_certificate_ids = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.216402] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.enable_certificate_validation = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.216610] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.enable_rbd_download = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.216810] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.endpoint_override = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.217011] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.insecure = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.217187] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.keyfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.217354] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.max_version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.217532] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.min_version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.217714] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.num_retries = 3 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.217888] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.rbd_ceph_conf = {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.218073] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.rbd_connect_timeout = 5 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.218255] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.rbd_pool = {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.218428] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.rbd_user = {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.218591] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.region_name = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.218753] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.retriable_status_codes = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.218913] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.service_name = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.219096] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.service_type = image {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.219266] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.split_loggers = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.219470] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.status_code_retries = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.219648] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.status_code_retry_delay = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.219820] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.timeout = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.220071] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.valid_interfaces = ['internal', 'public'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.220278] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.verify_glance_signatures = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.220453] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] glance.version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.220625] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] guestfs.debug = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.220794] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] mks.enabled = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.221201] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] mks.mksproxy_base_url = http://127.0.0.1:6090/ {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.221403] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] image_cache.manager_interval = 2400 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.221577] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] image_cache.precache_concurrency = 1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.221751] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] image_cache.remove_unused_base_images = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.221920] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] image_cache.remove_unused_original_minimum_age_seconds = 86400 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.222100] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] image_cache.remove_unused_resized_minimum_age_seconds = 3600 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.222318] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] image_cache.subdirectory_name = _base {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.222522] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.api_max_retries = 60 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.222690] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.api_retry_interval = 2 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.222890] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.auth_section = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.223086] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.auth_type = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.223256] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.cafile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.223422] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.certfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.223590] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.collect_timing = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.223756] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.conductor_group = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.223928] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.connect_retries = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.224105] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.connect_retry_delay = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.224269] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.endpoint_override = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.224436] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.insecure = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.224595] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.keyfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.224753] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.max_version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.224911] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.min_version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.225089] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.peer_list = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.225292] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.region_name = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.225472] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.retriable_status_codes = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.225640] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.serial_console_state_timeout = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.225812] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.service_name = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.226013] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.service_type = baremetal {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.226191] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.shard = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.226360] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.split_loggers = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.226520] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.status_code_retries = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.226681] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.status_code_retry_delay = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.226855] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.timeout = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.227048] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.valid_interfaces = ['internal', 'public'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.227217] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ironic.version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.227438] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] key_manager.backend = nova.keymgr.conf_key_mgr.ConfKeyManager {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.227625] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] key_manager.fixed_key = **** {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.227861] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican.auth_endpoint = http://localhost/identity/v3 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.227973] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican.barbican_api_version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.228177] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican.barbican_endpoint = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.228406] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican.barbican_endpoint_type = public {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.228579] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican.barbican_region_name = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.228743] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican.cafile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.228905] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican.certfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.229083] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican.collect_timing = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.229251] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican.insecure = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.229415] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican.keyfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.229601] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican.number_of_retries = 60 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.229784] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican.retry_delay = 1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.229953] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican.send_service_user_token = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.230134] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican.split_loggers = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.230327] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican.timeout = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.230499] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican.verify_ssl = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.230661] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican.verify_ssl_path = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.230831] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican_service_user.auth_section = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.231013] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican_service_user.auth_type = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.231224] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican_service_user.cafile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.231432] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican_service_user.certfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.231611] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican_service_user.collect_timing = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.231779] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican_service_user.insecure = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.231968] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican_service_user.keyfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.232121] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican_service_user.split_loggers = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.232286] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] barbican_service_user.timeout = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.232462] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vault.approle_role_id = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.232633] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vault.approle_secret_id = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.232798] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vault.cafile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.232960] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vault.certfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.233140] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vault.collect_timing = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.233308] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vault.insecure = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.233471] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vault.keyfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.233645] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vault.kv_mountpoint = secret {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.233811] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vault.kv_path = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.234033] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vault.kv_version = 2 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.234213] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vault.namespace = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.234410] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vault.root_token_id = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.234589] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vault.split_loggers = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.234755] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vault.ssl_ca_crt_file = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.234915] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vault.timeout = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.235093] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vault.use_ssl = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.235270] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vault.vault_url = http://127.0.0.1:8200 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.235451] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.auth_section = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.235622] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.auth_type = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.235786] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.cafile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.235948] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.certfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.236125] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.collect_timing = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.236290] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.connect_retries = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.236452] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.connect_retry_delay = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.236613] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.endpoint_override = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.236811] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.insecure = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.236996] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.keyfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.237174] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.max_version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.237357] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.min_version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.237537] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.region_name = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.237701] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.retriable_status_codes = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.237865] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.service_name = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.238052] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.service_type = identity {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.238229] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.split_loggers = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.238400] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.status_code_retries = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.238565] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.status_code_retry_delay = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.238725] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.timeout = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.238908] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.valid_interfaces = ['internal', 'public'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.239085] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] keystone.version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.239294] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.connection_uri = {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.239492] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.cpu_mode = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.239688] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.cpu_model_extra_flags = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.239893] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.cpu_models = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.240087] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.cpu_power_governor_high = performance {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.240288] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.cpu_power_governor_low = powersave {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.240489] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.cpu_power_management = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.240674] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.cpu_power_management_strategy = cpu_state {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.240846] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.device_detach_attempts = 8 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.241014] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.device_detach_timeout = 20 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.241194] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.disk_cachemodes = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.241366] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.disk_prefix = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.241536] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.enabled_perf_events = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.241727] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.file_backed_memory = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.241897] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.gid_maps = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.242071] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.hw_disk_discard = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.242235] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.hw_machine_type = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.242411] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.images_rbd_ceph_conf = {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.242597] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.images_rbd_glance_copy_poll_interval = 15 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.242797] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.images_rbd_glance_copy_timeout = 600 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.242975] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.images_rbd_glance_store_name = {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.243164] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.images_rbd_pool = rbd {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.243342] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.images_type = default {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.243539] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.images_volume_group = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.243714] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.inject_key = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.243882] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.inject_partition = -2 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.244060] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.inject_password = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.244242] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.iscsi_iface = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.244412] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.iser_use_multipath = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.244577] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.live_migration_bandwidth = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.244743] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.live_migration_completion_timeout = 800 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.244907] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.live_migration_downtime = 500 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.245091] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.live_migration_downtime_delay = 75 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.245260] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.live_migration_downtime_steps = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.245426] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.live_migration_inbound_addr = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.245626] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.live_migration_permit_auto_converge = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.245809] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.live_migration_permit_post_copy = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.245974] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.live_migration_scheme = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.246171] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.live_migration_timeout_action = abort {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.246358] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.live_migration_tunnelled = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.246549] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.live_migration_uri = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.246720] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.live_migration_with_native_tls = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.246884] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.max_queues = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.247070] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.mem_stats_period_seconds = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.247313] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.migration_inbound_addr = 10.180.1.21 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.247488] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.nfs_mount_options = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.247786] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.nfs_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.247960] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.num_aoe_discover_tries = 3 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.248145] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.num_iser_scan_tries = 5 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.248311] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.num_memory_encrypted_guests = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.248519] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.num_nvme_discover_tries = 5 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.248699] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.num_pcie_ports = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.248872] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.num_volume_scan_tries = 5 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.249052] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.pmem_namespaces = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.249222] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.quobyte_client_cfg = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.249549] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.quobyte_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.249744] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.rbd_connect_timeout = 5 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.249927] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.rbd_destroy_volume_retries = 12 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.250115] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.rbd_destroy_volume_retry_interval = 5 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.250317] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.rbd_secret_uuid = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.250484] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.rbd_user = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.250652] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.realtime_scheduler_priority = 1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.250828] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.remote_filesystem_transport = ssh {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.250991] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.rescue_image_id = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.251188] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.rescue_kernel_id = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.251418] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.rescue_ramdisk_id = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.251615] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.rng_dev_path = /dev/urandom {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.251783] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.rx_queue_size = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.251959] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.smbfs_mount_options = {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.252258] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.smbfs_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.252458] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.snapshot_compression = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.252641] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.snapshot_image_format = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.252866] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.snapshots_directory = /opt/stack/data/nova/instances/snapshots {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.253084] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.sparse_logical_volumes = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.253272] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.swtpm_enabled = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.253474] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.swtpm_group = tss {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.253652] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.swtpm_user = tss {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.253824] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.sysinfo_serial = unique {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.253986] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.tb_cache_size = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.254161] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.tx_queue_size = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.254332] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.uid_maps = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.254535] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.use_virtio_for_bridges = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.254728] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.virt_type = kvm {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.254902] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.volume_clear = zero {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.255086] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.volume_clear_size = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.255261] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.volume_use_multipath = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.255480] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.vzstorage_cache_path = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.255678] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.vzstorage_log_path = /var/log/vstorage/%(cluster_name)s/nova.log.gz {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.255855] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.vzstorage_mount_group = qemu {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.256069] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.vzstorage_mount_opts = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.256270] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.vzstorage_mount_perms = 0770 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.256555] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.vzstorage_mount_point_base = /opt/stack/data/n-cpu-1/mnt {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.256743] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.vzstorage_mount_user = stack {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.256908] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] libvirt.wait_soft_reboot_seconds = 120 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.257096] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.auth_section = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.257277] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.auth_type = password {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.257457] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.cafile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.257683] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.certfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.257862] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.collect_timing = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.258040] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.connect_retries = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.258243] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.connect_retry_delay = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.258457] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.default_floating_pool = public {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.258632] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.endpoint_override = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.258799] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.extension_sync_interval = 600 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.258993] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.http_retries = 3 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.259196] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.insecure = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.259367] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.keyfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.259530] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.max_version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.259706] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.metadata_proxy_shared_secret = **** {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.259870] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.min_version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.260056] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.ovs_bridge = br-int {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.260255] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.physnets = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.260457] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.region_name = RegionOne {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.260641] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.retriable_status_codes = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.260817] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.service_metadata_proxy = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.260982] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.service_name = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.261185] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.service_type = network {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.261388] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.split_loggers = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.261558] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.status_code_retries = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.261723] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.status_code_retry_delay = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.261922] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.timeout = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.262145] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.valid_interfaces = ['internal', 'public'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.262321] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] neutron.version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.262495] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] notifications.bdms_in_notifications = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.262674] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] notifications.default_level = INFO {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.262855] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] notifications.notification_format = unversioned {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.263034] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] notifications.notify_on_state_change = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.263218] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] notifications.versioned_notifications_topics = ['versioned_notifications'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.263423] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] pci.alias = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.263607] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] pci.device_spec = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.263776] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] pci.report_in_placement = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.263951] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.auth_section = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.264136] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.auth_type = password {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.264331] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.auth_url = http://10.180.1.21/identity {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.264508] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.cafile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.264671] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.certfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.264868] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.collect_timing = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.265057] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.connect_retries = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.265225] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.connect_retry_delay = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.265390] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.default_domain_id = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.265554] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.default_domain_name = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.265716] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.domain_id = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.265877] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.domain_name = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.266047] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.endpoint_override = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.266220] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.insecure = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.266419] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.keyfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.266587] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.max_version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.266749] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.min_version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.266920] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.password = **** {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.267093] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.project_domain_id = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.267279] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.project_domain_name = Default {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.267511] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.project_id = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.267709] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.project_name = service {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.267920] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.region_name = RegionOne {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.268108] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.retriable_status_codes = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.268277] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.service_name = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.268454] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.service_type = placement {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.268620] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.split_loggers = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.268783] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.status_code_retries = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.268947] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.status_code_retry_delay = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.269123] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.system_scope = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.269327] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.timeout = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.269503] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.trust_id = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.269948] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.user_domain_id = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.269948] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.user_domain_name = Default {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.270037] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.user_id = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.270188] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.username = placement {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.270385] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.valid_interfaces = ['internal', 'public'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.270552] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] placement.version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.270795] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] quota.cores = 20 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.270999] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] quota.count_usage_from_placement = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.271215] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] quota.driver = nova.quota.DbQuotaDriver {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.271408] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] quota.injected_file_content_bytes = 10240 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.271581] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] quota.injected_file_path_length = 255 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.271752] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] quota.injected_files = 5 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.271921] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] quota.instances = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.272100] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] quota.key_pairs = 100 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.272282] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] quota.metadata_items = 128 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.272483] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] quota.ram = 51200 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.272654] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] quota.recheck_quota = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.272826] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] quota.server_group_members = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.272995] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] quota.server_groups = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.273185] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] scheduler.discover_hosts_in_cells_interval = -1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.273355] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] scheduler.enable_isolated_aggregate_filtering = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.273521] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] scheduler.image_metadata_prefilter = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.273685] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] scheduler.limit_tenants_to_placement_aggregate = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.273881] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] scheduler.max_attempts = 3 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.274084] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] scheduler.max_placement_results = 1000 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.274262] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] scheduler.placement_aggregate_required_for_tenants = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.274432] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] scheduler.query_placement_for_image_type_support = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.274595] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] scheduler.query_placement_for_routed_network_aggregates = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.274771] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] scheduler.workers = 2 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.274952] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.aggregate_image_properties_isolation_namespace = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.275141] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.aggregate_image_properties_isolation_separator = . {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.275357] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.available_filters = ['nova.scheduler.filters.all_filters'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.275578] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.build_failure_weight_multiplier = 1000000.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.275758] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.cpu_weight_multiplier = 1.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.275929] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.cross_cell_move_weight_multiplier = 1000000.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.276109] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.disk_weight_multiplier = 1.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.276305] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.enabled_filters = ['ComputeFilter', 'ComputeCapabilitiesFilter', 'ImagePropertiesFilter', 'ServerGroupAntiAffinityFilter', 'ServerGroupAffinityFilter', 'SameHostFilter', 'DifferentHostFilter'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.276478] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.host_subset_size = 1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.276646] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.hypervisor_version_weight_multiplier = 1.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.276814] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.image_properties_default_architecture = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.277032] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.io_ops_weight_multiplier = -1.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.277215] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.isolated_hosts = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.277385] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.isolated_images = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.277551] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.max_instances_per_host = 50 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.277714] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.max_io_ops_per_host = 8 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.277890] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.num_instances_weight_multiplier = 0.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.278072] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.pci_in_placement = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.278245] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.pci_weight_multiplier = 1.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.278418] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.ram_weight_multiplier = 1.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.278587] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.restrict_isolated_hosts_to_isolated_images = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.278752] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.shuffle_best_same_weighed_hosts = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.278918] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.soft_affinity_weight_multiplier = 1.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.279097] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.soft_anti_affinity_weight_multiplier = 1.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.279265] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.track_instance_changes = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.279446] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] filter_scheduler.weight_classes = ['nova.scheduler.weights.all_weighers'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.279621] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] metrics.required = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.279810] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] metrics.weight_multiplier = 1.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.280013] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] metrics.weight_of_unavailable = -10000.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.280230] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] metrics.weight_setting = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.280557] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] serial_console.base_url = ws://127.0.0.1:6083/ {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.280737] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] serial_console.enabled = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.280920] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] serial_console.port_range = 10000:20000 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.281108] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] serial_console.proxyclient_address = 127.0.0.1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.281314] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] serial_console.serialproxy_host = 0.0.0.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.281521] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] serial_console.serialproxy_port = 6083 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.281701] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] service_user.auth_section = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.281877] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] service_user.auth_type = password {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.282056] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] service_user.cafile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.282221] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] service_user.certfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.282392] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] service_user.collect_timing = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.282557] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] service_user.insecure = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.282720] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] service_user.keyfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.282913] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] service_user.send_service_user_token = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.283112] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] service_user.split_loggers = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.283282] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] service_user.timeout = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.283460] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] spice.agent_enabled = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.283642] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] spice.enabled = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.283959] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] spice.html5proxy_base_url = http://127.0.0.1:6082/spice_auto.html {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.284165] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] spice.html5proxy_host = 0.0.0.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.284345] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] spice.html5proxy_port = 6082 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.284538] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] spice.image_compression = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.284716] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] spice.jpeg_compression = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.284883] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] spice.playback_compression = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.285069] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] spice.server_listen = 127.0.0.1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.285245] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] spice.server_proxyclient_address = 127.0.0.1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.285411] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] spice.streaming_mode = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.285572] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] spice.zlib_compression = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.285740] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] upgrade_levels.baseapi = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.285922] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] upgrade_levels.compute = auto {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.286134] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] upgrade_levels.conductor = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.286306] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] upgrade_levels.scheduler = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.286479] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vendordata_dynamic_auth.auth_section = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.286644] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vendordata_dynamic_auth.auth_type = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.286805] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vendordata_dynamic_auth.cafile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.286966] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vendordata_dynamic_auth.certfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.287146] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vendordata_dynamic_auth.collect_timing = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.287313] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vendordata_dynamic_auth.insecure = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.287477] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vendordata_dynamic_auth.keyfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.287684] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vendordata_dynamic_auth.split_loggers = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.287857] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vendordata_dynamic_auth.timeout = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.288048] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.api_retry_count = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.288218] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.ca_file = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.288396] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.cache_prefix = devstack-image-cache {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.288567] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.cluster_name = testcl1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.288737] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.connection_pool_size = 10 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.288901] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.console_delay_seconds = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.289143] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.datastore_regex = ^datastore.* {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.289378] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.host_ip = vc1.osci.c.eu-de-1.cloud.sap {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.289561] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.host_password = **** {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.289736] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.host_port = 443 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.289912] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.host_username = administrator@vsphere.local {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.290096] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.insecure = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.290292] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.integration_bridge = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.290467] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.maximum_objects = 100 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.290657] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.pbm_default_policy = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.290844] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.pbm_enabled = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.291025] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.pbm_wsdl_location = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.291226] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.serial_log_dir = /opt/vmware/vspc {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.291403] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.serial_port_proxy_uri = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.291567] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.serial_port_service_uri = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.291736] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.task_poll_interval = 0.5 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.291913] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.use_linked_clone = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.292110] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.vnc_keymap = en-us {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.292325] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.vnc_port = 5900 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.292513] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vmware.vnc_port_total = 10000 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.292704] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vnc.auth_schemes = ['none'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.292883] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vnc.enabled = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.293190] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vnc.novncproxy_base_url = http://127.0.0.1:6080/vnc_auto.html {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.293383] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vnc.novncproxy_host = 0.0.0.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.293560] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vnc.novncproxy_port = 6080 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.293746] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vnc.server_listen = 127.0.0.1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.293960] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vnc.server_proxyclient_address = 127.0.0.1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.294148] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vnc.vencrypt_ca_certs = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.294317] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vnc.vencrypt_client_cert = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.294478] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vnc.vencrypt_client_key = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.294659] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.disable_compute_service_check_for_ffu = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.294824] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.disable_deep_image_inspection = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.294985] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.disable_fallback_pcpu_query = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.295166] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.disable_group_policy_check_upcall = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.295366] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.disable_libvirt_livesnapshot = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.295548] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.disable_rootwrap = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.295713] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.enable_numa_live_migration = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.295875] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.enable_qemu_monitor_announce_self = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.296048] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.ensure_libvirt_rbd_instance_dir_cleanup = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.296214] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.handle_virt_lifecycle_events = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.296379] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.libvirt_disable_apic = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.296539] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.never_download_image_if_on_rbd = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.296701] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.qemu_monitor_announce_self_count = 3 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.296934] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.qemu_monitor_announce_self_interval = 1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.297143] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.reserve_disk_resource_for_image_cache = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.297316] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.skip_cpu_compare_at_startup = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.297484] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.skip_cpu_compare_on_dest = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.297647] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.skip_hypervisor_version_check_on_lm = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.297812] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.skip_reserve_in_use_ironic_nodes = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.297973] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.unified_limits_count_pcpu_as_vcpu = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.298156] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] workarounds.wait_for_vif_plugged_event_during_hard_reboot = [] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.298339] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] wsgi.api_paste_config = /etc/nova/api-paste.ini {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.298542] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] wsgi.client_socket_timeout = 900 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.298721] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] wsgi.default_pool_size = 1000 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.298891] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] wsgi.keep_alive = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.299069] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] wsgi.max_header_line = 16384 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.299241] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] wsgi.secure_proxy_ssl_header = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.299408] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] wsgi.ssl_ca_file = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.299570] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] wsgi.ssl_cert_file = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.299733] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] wsgi.ssl_key_file = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.299900] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] wsgi.tcp_keepidle = 600 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.300122] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] wsgi.wsgi_log_format = %(client_ip)s "%(request_line)s" status: %(status_code)s len: %(body_length)s time: %(wall_seconds).7f {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.300329] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] zvm.ca_file = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.300503] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] zvm.cloud_connector_url = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.300787] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] zvm.image_tmp_path = /opt/stack/data/n-cpu-1/images {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.300963] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] zvm.reachable_timeout = 300 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.301172] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_policy.enforce_new_defaults = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.301374] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_policy.enforce_scope = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.301582] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_policy.policy_default_rule = default {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.301782] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_policy.policy_dirs = ['policy.d'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.301962] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_policy.policy_file = policy.yaml {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.302151] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_policy.remote_content_type = application/x-www-form-urlencoded {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.302320] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_policy.remote_ssl_ca_crt_file = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.302482] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_policy.remote_ssl_client_crt_file = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.302643] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_policy.remote_ssl_client_key_file = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.302807] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_policy.remote_ssl_verify_server_crt = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.302993] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_versionedobjects.fatal_exception_format_errors = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.303216] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_middleware.http_basic_auth_user_file = /etc/htpasswd {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.303405] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] profiler.connection_string = messaging:// {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.303575] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] profiler.enabled = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.303748] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] profiler.es_doc_type = notification {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.303915] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] profiler.es_scroll_size = 10000 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.304096] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] profiler.es_scroll_time = 2m {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.304266] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] profiler.filter_error_trace = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.304438] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] profiler.hmac_keys = **** {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.304678] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] profiler.sentinel_service_name = mymaster {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.304864] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] profiler.socket_timeout = 0.1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.305045] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] profiler.trace_requests = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.305214] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] profiler.trace_sqlalchemy = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.305396] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] profiler_jaeger.process_tags = {} {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.305561] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] profiler_jaeger.service_name_prefix = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.305725] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] profiler_otlp.service_name_prefix = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.305892] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] remote_debug.host = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.306067] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] remote_debug.port = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.306286] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.amqp_auto_delete = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.306462] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.amqp_durable_queues = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.306631] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.conn_pool_min_size = 2 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.306796] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.conn_pool_ttl = 1200 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.306961] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.direct_mandatory_flag = True {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.307140] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.enable_cancel_on_failover = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.307307] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.heartbeat_in_pthread = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.307475] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.heartbeat_rate = 3 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.307642] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.heartbeat_timeout_threshold = 60 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.307852] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.hostname = devstack {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.308036] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.kombu_compression = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.308216] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.kombu_failover_strategy = round-robin {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.308389] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.kombu_missing_consumer_retry_timeout = 60 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.308561] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.kombu_reconnect_delay = 1.0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.308733] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.processname = nova-compute {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.308902] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.rabbit_ha_queues = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.309077] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.rabbit_interval_max = 30 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.309281] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.rabbit_login_method = AMQPLAIN {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.309475] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.rabbit_qos_prefetch_count = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.309645] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.rabbit_quorum_delivery_limit = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.309814] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.rabbit_quorum_max_memory_bytes = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.309980] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.rabbit_quorum_max_memory_length = 0 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.310180] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.rabbit_quorum_queue = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.310353] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.rabbit_retry_backoff = 2 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.310520] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.rabbit_retry_interval = 1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.310682] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.rabbit_stream_fanout = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.310885] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.rabbit_transient_queues_ttl = 1800 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.311079] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.rabbit_transient_quorum_queue = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.311280] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.rpc_conn_pool_size = 30 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.311460] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.ssl = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.311637] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.ssl_ca_file = {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.311810] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.ssl_cert_file = {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.311973] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.ssl_enforce_fips_mode = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.312161] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.ssl_key_file = {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.312370] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.ssl_version = {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.312602] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_rabbit.use_queue_manager = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.312852] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_notifications.driver = ['messagingv2'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.312974] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_notifications.retry = -1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.313176] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_notifications.topics = ['notifications'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.313358] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_messaging_notifications.transport_url = **** {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.313533] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.auth_section = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.313697] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.auth_type = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.313860] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.cafile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.314059] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.certfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.314240] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.collect_timing = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.314404] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.connect_retries = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.314562] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.connect_retry_delay = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.314722] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.endpoint_id = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.314877] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.endpoint_override = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.315051] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.insecure = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.315213] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.keyfile = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.315374] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.max_version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.315568] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.min_version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.315740] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.region_name = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.315904] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.retriable_status_codes = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.316076] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.service_name = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.316240] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.service_type = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.316406] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.split_loggers = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.316567] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.status_code_retries = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.316726] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.status_code_retry_delay = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.316884] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.timeout = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.317085] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.valid_interfaces = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.317259] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_limit.version = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.317429] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_reports.file_event_handler = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.317595] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_reports.file_event_handler_interval = 1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.317754] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] oslo_reports.log_dir = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.317927] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vif_plug_linux_bridge_privileged.capabilities = [12] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.318103] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vif_plug_linux_bridge_privileged.group = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.318266] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vif_plug_linux_bridge_privileged.helper_command = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.318440] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vif_plug_linux_bridge_privileged.logger_name = oslo_privsep.daemon {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.318644] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vif_plug_linux_bridge_privileged.thread_pool_size = 8 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.318814] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vif_plug_linux_bridge_privileged.user = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.318987] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vif_plug_ovs_privileged.capabilities = [12, 1] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.319166] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vif_plug_ovs_privileged.group = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.319331] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vif_plug_ovs_privileged.helper_command = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.319499] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vif_plug_ovs_privileged.logger_name = oslo_privsep.daemon {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.319664] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vif_plug_ovs_privileged.thread_pool_size = 8 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.319824] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] vif_plug_ovs_privileged.user = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.320037] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] os_vif_linux_bridge.flat_interface = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.320288] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] os_vif_linux_bridge.forward_bridge_interface = ['all'] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.320482] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] os_vif_linux_bridge.iptables_bottom_regex = {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.320658] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] os_vif_linux_bridge.iptables_drop_action = DROP {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.320833] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] os_vif_linux_bridge.iptables_top_regex = {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.321009] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] os_vif_linux_bridge.network_device_mtu = 1500 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.321206] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] os_vif_linux_bridge.use_ipv6 = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.321385] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] os_vif_linux_bridge.vlan_interface = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.321569] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] os_vif_ovs.default_qos_type = linux-noop {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.321786] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] os_vif_ovs.isolate_vif = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.321979] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] os_vif_ovs.network_device_mtu = 1500 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.322170] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] os_vif_ovs.ovs_vsctl_timeout = 120 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.322347] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] os_vif_ovs.ovsdb_connection = tcp:127.0.0.1:6640 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.322523] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] os_vif_ovs.ovsdb_interface = native {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.322688] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] os_vif_ovs.per_port_bridge = False {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.322855] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] os_brick.lock_path = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.323037] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] os_brick.wait_mpath_device_attempts = 4 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.323214] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] os_brick.wait_mpath_device_interval = 1 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.323392] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] privsep_osbrick.capabilities = [21] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.323550] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] privsep_osbrick.group = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.323709] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] privsep_osbrick.helper_command = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.323875] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] privsep_osbrick.logger_name = os_brick.privileged {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.324050] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] privsep_osbrick.thread_pool_size = 8 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.324215] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] privsep_osbrick.user = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.324390] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] nova_sys_admin.capabilities = [0, 1, 2, 3, 12, 21] {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.324551] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] nova_sys_admin.group = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.324759] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] nova_sys_admin.helper_command = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.324938] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] nova_sys_admin.logger_name = oslo_privsep.daemon {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.325118] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] nova_sys_admin.thread_pool_size = 8 {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.325283] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] nova_sys_admin.user = None {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 535.325420] env[67752]: DEBUG oslo_service.service [None req-4d35fe4b-aeb0-4a67-a49a-079dd53634fe None None] ******************************************************************************** {{(pid=67752) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2624}} [ 535.325836] env[67752]: INFO nova.service [-] Starting compute node (version 0.0.1) [ 535.338725] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Getting list of instances from cluster (obj){ [ 535.338725] env[67752]: value = "domain-c8" [ 535.338725] env[67752]: _type = "ClusterComputeResource" [ 535.338725] env[67752]: } {{(pid=67752) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 535.339966] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4254ec1b-5ae8-4032-a72f-9112f2ffa036 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 535.348977] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Got total of 0 instances {{(pid=67752) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 535.349536] env[67752]: WARNING nova.virt.vmwareapi.driver [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] The vmwareapi driver is not tested by the OpenStack project nor does it have clear maintainer(s) and thus its quality can not be ensured. It should be considered experimental and may be removed in a future release. If you are using the driver in production please let us know via the openstack-discuss mailing list. [ 535.350061] env[67752]: INFO nova.virt.node [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Generated node identity c7141b98-bb6c-46bd-b37f-a8e913940199 [ 535.350327] env[67752]: INFO nova.virt.node [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Wrote node identity c7141b98-bb6c-46bd-b37f-a8e913940199 to /opt/stack/data/n-cpu-1/compute_id [ 535.362571] env[67752]: WARNING nova.compute.manager [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Compute nodes ['c7141b98-bb6c-46bd-b37f-a8e913940199'] for host cpu-1 were not found in the database. If this is the first time this service is starting on this host, then you can ignore this warning. [ 535.398187] env[67752]: INFO nova.compute.manager [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Looking for unclaimed instances stuck in BUILDING status for nodes managed by this host [ 535.423203] env[67752]: WARNING nova.compute.manager [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] No compute node record found for host cpu-1. If this is the first time this service is starting on this host, then you can ignore this warning.: nova.exception_Remote.ComputeHostNotFound_Remote: Compute host cpu-1 could not be found. [ 535.423690] env[67752]: DEBUG oslo_concurrency.lockutils [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.423932] env[67752]: DEBUG oslo_concurrency.lockutils [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.424108] env[67752]: DEBUG oslo_concurrency.lockutils [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 535.424266] env[67752]: DEBUG nova.compute.resource_tracker [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 535.425359] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f82d701-ad6a-499b-b773-f248b86805e8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 535.433638] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-72356ef8-10d5-4418-b182-4cf573618750 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 535.447401] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f0d50be-7f72-4119-b94d-98006f7ac272 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 535.454051] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-53856632-7367-47db-afd8-72bc987af329 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 535.483070] env[67752]: DEBUG nova.compute.resource_tracker [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181003MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 535.483070] env[67752]: DEBUG oslo_concurrency.lockutils [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.483070] env[67752]: DEBUG oslo_concurrency.lockutils [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.494850] env[67752]: WARNING nova.compute.resource_tracker [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] No compute node record for cpu-1:c7141b98-bb6c-46bd-b37f-a8e913940199: nova.exception_Remote.ComputeHostNotFound_Remote: Compute host c7141b98-bb6c-46bd-b37f-a8e913940199 could not be found. [ 535.507493] env[67752]: INFO nova.compute.resource_tracker [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Compute node record created for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 with uuid: c7141b98-bb6c-46bd-b37f-a8e913940199 [ 535.558792] env[67752]: DEBUG nova.compute.resource_tracker [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Total usable vcpus: 48, total allocated vcpus: 0 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 535.558964] env[67752]: DEBUG nova.compute.resource_tracker [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=512MB phys_disk=200GB used_disk=0GB total_vcpus=48 used_vcpus=0 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 535.659309] env[67752]: INFO nova.scheduler.client.report [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] [req-2caefca9-e958-4485-9fa9-7742569f66ec] Created resource provider record via placement API for resource provider with UUID c7141b98-bb6c-46bd-b37f-a8e913940199 and name domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28. [ 535.676883] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1b213530-e925-4eac-b163-d8c645acc7d3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 535.684378] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6a521028-13d9-411f-91d5-f055d2b5387f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 535.713224] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-37e5f306-59c0-458c-9b50-147d742d7ccd {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 535.720230] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4864840-2af4-4583-8cb6-287f5dba21b8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 535.733646] env[67752]: DEBUG nova.compute.provider_tree [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 535.774121] env[67752]: DEBUG nova.scheduler.client.report [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Updated inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with generation 0 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:957}} [ 535.774462] env[67752]: DEBUG nova.compute.provider_tree [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Updating resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 generation from 0 to 1 during operation: update_inventory {{(pid=67752) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 535.774708] env[67752]: DEBUG nova.compute.provider_tree [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 535.821089] env[67752]: DEBUG nova.compute.provider_tree [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Updating resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 generation from 1 to 2 during operation: update_traits {{(pid=67752) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 535.839403] env[67752]: DEBUG nova.compute.resource_tracker [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 535.839592] env[67752]: DEBUG oslo_concurrency.lockutils [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.356s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 535.839756] env[67752]: DEBUG nova.service [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Creating RPC server for service compute {{(pid=67752) start /opt/stack/nova/nova/service.py:182}} [ 535.852850] env[67752]: DEBUG nova.service [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] Join ServiceGroup membership for this service compute {{(pid=67752) start /opt/stack/nova/nova/service.py:199}} [ 535.853048] env[67752]: DEBUG nova.servicegroup.drivers.db [None req-cc46d201-d6df-43f1-992b-75a324c9d5a3 None None] DB_Driver: join new ServiceGroup member cpu-1 to the compute group, service = {{(pid=67752) join /opt/stack/nova/nova/servicegroup/drivers/db.py:44}} [ 545.156521] env[67752]: DEBUG dbcounter [-] [67752] Writing DB stats nova_cell1:SELECT=1 {{(pid=67752) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 545.158014] env[67752]: DEBUG dbcounter [-] [67752] Writing DB stats nova_cell0:SELECT=1 {{(pid=67752) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 573.334789] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Acquiring lock "0085fe38-e5ec-44f9-b51b-c54a9f05b60f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 573.334789] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Lock "0085fe38-e5ec-44f9-b51b-c54a9f05b60f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 573.360438] env[67752]: DEBUG nova.compute.manager [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 573.490417] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 573.490417] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 573.492761] env[67752]: INFO nova.compute.claims [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 573.656837] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0da3b64-0186-461e-84b9-ce3fd94655c6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 573.667852] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d16a9d29-d6a5-4b1c-a67b-c9979476e8be {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 573.701952] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-241c9c4a-cac7-43f2-985a-90b403f34b2e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 573.709597] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d7b06fb-75bd-4148-81fb-3c8594a9f4bc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 573.723228] env[67752]: DEBUG nova.compute.provider_tree [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 573.735057] env[67752]: DEBUG nova.scheduler.client.report [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 573.754454] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.265s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 573.755032] env[67752]: DEBUG nova.compute.manager [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 573.793847] env[67752]: DEBUG nova.compute.utils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 573.796522] env[67752]: DEBUG nova.compute.manager [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 573.796819] env[67752]: DEBUG nova.network.neutron [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 573.812855] env[67752]: DEBUG nova.compute.manager [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 573.908503] env[67752]: DEBUG nova.compute.manager [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 576.069565] env[67752]: DEBUG nova.virt.hardware [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 576.069565] env[67752]: DEBUG nova.virt.hardware [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 576.069951] env[67752]: DEBUG nova.virt.hardware [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 576.069951] env[67752]: DEBUG nova.virt.hardware [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 576.070391] env[67752]: DEBUG nova.virt.hardware [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 576.070586] env[67752]: DEBUG nova.virt.hardware [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 576.070806] env[67752]: DEBUG nova.virt.hardware [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 576.070988] env[67752]: DEBUG nova.virt.hardware [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 576.071448] env[67752]: DEBUG nova.virt.hardware [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 576.072624] env[67752]: DEBUG nova.virt.hardware [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 576.072835] env[67752]: DEBUG nova.virt.hardware [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 576.073851] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-979fd6ea-29a9-485e-82d1-14fbe8209925 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 576.086897] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f3171f9-37db-4c87-9065-ab11f6625187 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 576.113870] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3c37d8db-550d-4a12-9750-7c0042ac6e43 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 576.548360] env[67752]: DEBUG nova.policy [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'e0034c63c62f42fabc3ff9027d0fdb5b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '985744beaef54bab978b5be74514cd0d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 578.854651] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._sync_power_states {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 578.874191] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Getting list of instances from cluster (obj){ [ 578.874191] env[67752]: value = "domain-c8" [ 578.874191] env[67752]: _type = "ClusterComputeResource" [ 578.874191] env[67752]: } {{(pid=67752) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 578.875576] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56fb519e-73bc-4dd6-8342-183ef09f1262 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 578.887419] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Got total of 0 instances {{(pid=67752) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 578.887419] env[67752]: WARNING nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] While synchronizing instance power states, found 1 instances in the database and 0 instances on the hypervisor. [ 578.888113] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid 0085fe38-e5ec-44f9-b51b-c54a9f05b60f {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 578.888878] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "0085fe38-e5ec-44f9-b51b-c54a9f05b60f" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 578.889243] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._cleanup_running_deleted_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 578.889694] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Getting list of instances from cluster (obj){ [ 578.889694] env[67752]: value = "domain-c8" [ 578.889694] env[67752]: _type = "ClusterComputeResource" [ 578.889694] env[67752]: } {{(pid=67752) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 578.894462] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ce4e1ce-f5c6-4ce0-a3cd-5fd0b5807e52 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 578.905495] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Got total of 0 instances {{(pid=67752) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 579.111030] env[67752]: DEBUG nova.network.neutron [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Successfully created port: 5d7f3c98-5e74-4893-ba90-683172689749 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 582.011585] env[67752]: DEBUG oslo_concurrency.lockutils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Acquiring lock "b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 582.011585] env[67752]: DEBUG oslo_concurrency.lockutils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Lock "b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 582.032644] env[67752]: DEBUG nova.compute.manager [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 582.142471] env[67752]: DEBUG oslo_concurrency.lockutils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 582.142789] env[67752]: DEBUG oslo_concurrency.lockutils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 582.147911] env[67752]: INFO nova.compute.claims [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 582.381966] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-180ecf5b-9188-4642-873d-bd8b975ec179 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.403029] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5569bca2-e911-407e-91f5-2ad278e5d30b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.458181] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-84871fcc-283e-4961-8ebc-1f78b2e45821 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.466220] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e38c32d-d2cb-4fd6-b499-1728c50644e7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.486824] env[67752]: DEBUG nova.compute.provider_tree [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 582.507011] env[67752]: DEBUG nova.scheduler.client.report [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 582.531416] env[67752]: DEBUG oslo_concurrency.lockutils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.388s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 582.531915] env[67752]: DEBUG nova.compute.manager [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 582.602546] env[67752]: DEBUG nova.compute.utils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 582.605209] env[67752]: DEBUG nova.compute.manager [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 582.605594] env[67752]: DEBUG nova.network.neutron [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 582.623054] env[67752]: DEBUG nova.compute.manager [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 582.720358] env[67752]: DEBUG nova.compute.manager [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 582.763414] env[67752]: DEBUG nova.virt.hardware [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 582.763414] env[67752]: DEBUG nova.virt.hardware [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 582.763414] env[67752]: DEBUG nova.virt.hardware [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 582.763730] env[67752]: DEBUG nova.virt.hardware [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 582.763730] env[67752]: DEBUG nova.virt.hardware [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 582.763730] env[67752]: DEBUG nova.virt.hardware [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 582.764566] env[67752]: DEBUG nova.virt.hardware [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 582.764566] env[67752]: DEBUG nova.virt.hardware [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 582.764566] env[67752]: DEBUG nova.virt.hardware [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 582.764566] env[67752]: DEBUG nova.virt.hardware [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 582.764823] env[67752]: DEBUG nova.virt.hardware [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 582.766157] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-335f7f26-e0ba-4d95-a2d8-820978801580 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.779375] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd609c26-27a0-4f9a-96a2-ee4a2a19f308 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 582.998323] env[67752]: DEBUG nova.network.neutron [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Successfully updated port: 5d7f3c98-5e74-4893-ba90-683172689749 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 583.032090] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Acquiring lock "refresh_cache-0085fe38-e5ec-44f9-b51b-c54a9f05b60f" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 583.032396] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Acquired lock "refresh_cache-0085fe38-e5ec-44f9-b51b-c54a9f05b60f" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 583.032396] env[67752]: DEBUG nova.network.neutron [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 583.178793] env[67752]: DEBUG nova.policy [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '749fc4a2ee74425f8d0c83940e29ab11', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'cb995b20e0644427b10975527ced5266', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 583.274502] env[67752]: DEBUG nova.network.neutron [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 584.407091] env[67752]: DEBUG nova.network.neutron [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Updating instance_info_cache with network_info: [{"id": "5d7f3c98-5e74-4893-ba90-683172689749", "address": "fa:16:3e:46:85:5e", "network": {"id": "8754f810-4855-45bb-9a57-6c1511f0722f", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.180", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "c43e0a2618b34312898e0e4bb014b17c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "983826cf-6390-4ec6-bf97-30a1060947fc", "external-id": "nsx-vlan-transportzone-367", "segmentation_id": 367, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5d7f3c98-5e", "ovs_interfaceid": "5d7f3c98-5e74-4893-ba90-683172689749", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 584.422728] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Releasing lock "refresh_cache-0085fe38-e5ec-44f9-b51b-c54a9f05b60f" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 584.423043] env[67752]: DEBUG nova.compute.manager [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Instance network_info: |[{"id": "5d7f3c98-5e74-4893-ba90-683172689749", "address": "fa:16:3e:46:85:5e", "network": {"id": "8754f810-4855-45bb-9a57-6c1511f0722f", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.180", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "c43e0a2618b34312898e0e4bb014b17c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "983826cf-6390-4ec6-bf97-30a1060947fc", "external-id": "nsx-vlan-transportzone-367", "segmentation_id": 367, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5d7f3c98-5e", "ovs_interfaceid": "5d7f3c98-5e74-4893-ba90-683172689749", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 584.424082] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:46:85:5e', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '983826cf-6390-4ec6-bf97-30a1060947fc', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '5d7f3c98-5e74-4893-ba90-683172689749', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 584.443037] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Creating folder: OpenStack. Parent ref: group-v4. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 584.443482] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-f3973bcc-2bd0-44d0-b2eb-150a203a7e20 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 584.458786] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Created folder: OpenStack in parent group-v4. [ 584.458976] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Creating folder: Project (985744beaef54bab978b5be74514cd0d). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 584.459254] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-5fdb65b3-0662-47ab-a236-c98f33ca01bc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 584.477065] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Created folder: Project (985744beaef54bab978b5be74514cd0d) in parent group-v639722. [ 584.477065] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Creating folder: Instances. Parent ref: group-v639723. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 584.477065] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-7c87c2f3-a5c2-4773-a2f8-3f2c5b0b3bfa {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 584.485582] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Created folder: Instances in parent group-v639723. [ 584.485645] env[67752]: DEBUG oslo.service.loopingcall [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 584.486114] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 584.486114] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-814309c5-36db-4c86-872d-7ac516c05e8f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 584.508377] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 584.508377] env[67752]: value = "task-3199595" [ 584.508377] env[67752]: _type = "Task" [ 584.508377] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 584.518762] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199595, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 585.024119] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199595, 'name': CreateVM_Task, 'duration_secs': 0.386732} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 585.024119] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 585.103020] env[67752]: DEBUG oslo_vmware.service [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-edc0ce7f-9364-44b5-8957-4bbe2928d473 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 585.107845] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 585.108189] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 585.108913] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 585.109504] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-a55161c3-53cb-4b45-8887-9045eff7f780 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 585.116385] env[67752]: DEBUG oslo_vmware.api [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Waiting for the task: (returnval){ [ 585.116385] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5255a0d1-e3e5-d530-4039-8266326dcd5b" [ 585.116385] env[67752]: _type = "Task" [ 585.116385] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 585.124948] env[67752]: DEBUG oslo_vmware.api [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5255a0d1-e3e5-d530-4039-8266326dcd5b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 585.631152] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 585.631540] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 585.631819] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 585.632086] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 585.634269] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 585.634269] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-ebe4ee0d-7db5-4322-845f-094a333b98bf {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 585.644765] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 585.644765] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 585.645095] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-391c9501-1938-4e18-bb26-495d383620b7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 585.652514] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c4022df4-175b-4678-9dd6-5572e29fd186 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 585.660013] env[67752]: DEBUG oslo_vmware.api [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Waiting for the task: (returnval){ [ 585.660013] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5228c72f-77c9-100d-81d9-d8872f3c9da1" [ 585.660013] env[67752]: _type = "Task" [ 585.660013] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 585.671300] env[67752]: DEBUG oslo_vmware.api [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5228c72f-77c9-100d-81d9-d8872f3c9da1, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 585.692048] env[67752]: DEBUG nova.compute.manager [req-b6a45a71-3186-4cc1-bc27-f3bc11e86898 req-e04f537f-e875-4082-99a4-13ef96925478 service nova] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Received event network-vif-plugged-5d7f3c98-5e74-4893-ba90-683172689749 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 585.692294] env[67752]: DEBUG oslo_concurrency.lockutils [req-b6a45a71-3186-4cc1-bc27-f3bc11e86898 req-e04f537f-e875-4082-99a4-13ef96925478 service nova] Acquiring lock "0085fe38-e5ec-44f9-b51b-c54a9f05b60f-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 585.692783] env[67752]: DEBUG oslo_concurrency.lockutils [req-b6a45a71-3186-4cc1-bc27-f3bc11e86898 req-e04f537f-e875-4082-99a4-13ef96925478 service nova] Lock "0085fe38-e5ec-44f9-b51b-c54a9f05b60f-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 585.693051] env[67752]: DEBUG oslo_concurrency.lockutils [req-b6a45a71-3186-4cc1-bc27-f3bc11e86898 req-e04f537f-e875-4082-99a4-13ef96925478 service nova] Lock "0085fe38-e5ec-44f9-b51b-c54a9f05b60f-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 585.693278] env[67752]: DEBUG nova.compute.manager [req-b6a45a71-3186-4cc1-bc27-f3bc11e86898 req-e04f537f-e875-4082-99a4-13ef96925478 service nova] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] No waiting events found dispatching network-vif-plugged-5d7f3c98-5e74-4893-ba90-683172689749 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 585.693472] env[67752]: WARNING nova.compute.manager [req-b6a45a71-3186-4cc1-bc27-f3bc11e86898 req-e04f537f-e875-4082-99a4-13ef96925478 service nova] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Received unexpected event network-vif-plugged-5d7f3c98-5e74-4893-ba90-683172689749 for instance with vm_state building and task_state spawning. [ 585.755544] env[67752]: DEBUG nova.network.neutron [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Successfully created port: 3071faa7-0c8f-4f79-b84f-371bd0685c6d {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 586.174325] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 586.174908] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Creating directory with path [datastore2] vmware_temp/b43599fd-9897-4c19-b277-723581846f65/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 586.175284] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-e9231efb-5333-4c81-b0c0-4d7676be6976 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 586.200348] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Created directory with path [datastore2] vmware_temp/b43599fd-9897-4c19-b277-723581846f65/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 586.200557] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Fetch image to [datastore2] vmware_temp/b43599fd-9897-4c19-b277-723581846f65/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 586.200725] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/b43599fd-9897-4c19-b277-723581846f65/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 586.201552] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-281abe22-0287-4cf3-bd25-99a990c80602 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 586.210017] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf715e75-05c0-46c0-8a4a-c1ca7fa48b2d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 586.223969] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa0fb493-68e6-4cd5-a913-c7220155e48c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 586.264816] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0d4a8f8-0ab7-4ad9-a142-23c09d4b9fae {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 586.275116] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-9df0db13-50ae-43b8-a63b-fb0c3d2ea7c2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 586.373760] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 586.439904] env[67752]: DEBUG oslo_vmware.rw_handles [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/b43599fd-9897-4c19-b277-723581846f65/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 586.510764] env[67752]: DEBUG oslo_vmware.rw_handles [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 586.510971] env[67752]: DEBUG oslo_vmware.rw_handles [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Closing write handle for https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/b43599fd-9897-4c19-b277-723581846f65/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 588.062349] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Acquiring lock "c563b7ed-4fb3-4fb0-8350-789fbe4d53a2" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 588.062731] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Lock "c563b7ed-4fb3-4fb0-8350-789fbe4d53a2" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 588.074997] env[67752]: DEBUG nova.compute.manager [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 588.160455] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 588.160920] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 588.162865] env[67752]: INFO nova.compute.claims [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 588.337275] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c3540b4-9a7f-4acf-b720-b685156736c9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.347696] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-80126a6f-769f-4891-9ecc-e97743a830c3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.396777] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eefb0f8c-914d-4f7e-ba1d-a2eb6b6d750c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.406429] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-55e7a813-edad-4d9e-af40-3b8b8bb44467 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.424369] env[67752]: DEBUG nova.compute.provider_tree [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 588.436724] env[67752]: DEBUG nova.scheduler.client.report [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 588.464992] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.304s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 588.465501] env[67752]: DEBUG nova.compute.manager [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 588.513811] env[67752]: DEBUG nova.compute.utils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 588.515727] env[67752]: DEBUG nova.compute.manager [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Not allocating networking since 'none' was specified. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1968}} [ 588.529332] env[67752]: DEBUG nova.compute.manager [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 588.641502] env[67752]: DEBUG nova.compute.manager [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 588.657385] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Acquiring lock "fc4da5fa-1921-44e3-9100-042391d4ccb9" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 588.657667] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Lock "fc4da5fa-1921-44e3-9100-042391d4ccb9" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 588.679467] env[67752]: DEBUG nova.compute.manager [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 588.685088] env[67752]: DEBUG nova.virt.hardware [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 588.685340] env[67752]: DEBUG nova.virt.hardware [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 588.685501] env[67752]: DEBUG nova.virt.hardware [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 588.685788] env[67752]: DEBUG nova.virt.hardware [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 588.685882] env[67752]: DEBUG nova.virt.hardware [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 588.685972] env[67752]: DEBUG nova.virt.hardware [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 588.686491] env[67752]: DEBUG nova.virt.hardware [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 588.686659] env[67752]: DEBUG nova.virt.hardware [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 588.687299] env[67752]: DEBUG nova.virt.hardware [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 588.687360] env[67752]: DEBUG nova.virt.hardware [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 588.687817] env[67752]: DEBUG nova.virt.hardware [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 588.688921] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d6507f5-b9b4-4fdb-8624-64512e10d671 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.701275] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e164e2a3-3e41-418a-8a4f-d8e304631b54 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.718417] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Instance VIF info [] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 588.722355] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Creating folder: Project (c36cd7a50e9b46168e24ee64ad66fab5). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 588.723469] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-07e6add3-85dd-49ad-9744-b02f76741724 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.741119] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Created folder: Project (c36cd7a50e9b46168e24ee64ad66fab5) in parent group-v639722. [ 588.741789] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Creating folder: Instances. Parent ref: group-v639729. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 588.741789] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-b3988fca-7225-400f-84e8-0654a2be129f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.750722] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 588.750974] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 588.752993] env[67752]: INFO nova.compute.claims [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 588.760018] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Created folder: Instances in parent group-v639729. [ 588.760018] env[67752]: DEBUG oslo.service.loopingcall [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 588.760018] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 588.760018] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-9f07dc5e-d521-4e0f-8a8f-79e32b909e3e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.781731] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Acquiring lock "b5eee51b-73b8-4e17-98a4-e108a5b21892" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 588.782037] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Lock "b5eee51b-73b8-4e17-98a4-e108a5b21892" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 588.788863] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 588.788863] env[67752]: value = "task-3199602" [ 588.788863] env[67752]: _type = "Task" [ 588.788863] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 588.793935] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199602, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 588.808840] env[67752]: DEBUG nova.compute.manager [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 588.891945] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 588.919680] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d7fb227b-10a6-4726-b608-88f53755b849 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.931580] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4320dbd-d682-49e8-a70b-bb3889c3f1d4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.972655] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b9d7f64c-1a22-4f62-97f1-9ed81fa078e5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.981278] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-161875a3-5f27-4305-bbf9-57b6e5d6e36c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 588.997330] env[67752]: DEBUG nova.compute.provider_tree [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 589.017431] env[67752]: DEBUG nova.scheduler.client.report [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 589.046191] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.295s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 589.046801] env[67752]: DEBUG nova.compute.manager [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 589.050008] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.158s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 589.051880] env[67752]: INFO nova.compute.claims [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 589.107567] env[67752]: DEBUG nova.compute.utils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 589.109318] env[67752]: DEBUG nova.compute.manager [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 589.109503] env[67752]: DEBUG nova.network.neutron [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 589.132170] env[67752]: DEBUG nova.compute.manager [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 589.235982] env[67752]: DEBUG nova.compute.manager [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 589.267777] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-05e75ed2-072a-46de-a24a-27e13f3837a8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.274581] env[67752]: DEBUG nova.virt.hardware [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 589.274902] env[67752]: DEBUG nova.virt.hardware [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 589.275207] env[67752]: DEBUG nova.virt.hardware [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 589.275292] env[67752]: DEBUG nova.virt.hardware [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 589.275437] env[67752]: DEBUG nova.virt.hardware [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 589.275614] env[67752]: DEBUG nova.virt.hardware [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 589.275832] env[67752]: DEBUG nova.virt.hardware [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 589.275988] env[67752]: DEBUG nova.virt.hardware [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 589.276195] env[67752]: DEBUG nova.virt.hardware [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 589.276359] env[67752]: DEBUG nova.virt.hardware [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 589.276529] env[67752]: DEBUG nova.virt.hardware [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 589.278118] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67151fc2-c51c-4373-a96c-9e72322ff58c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.298181] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-30163bc0-e37e-4c46-88e1-f3f214033dc7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.304551] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-986f618d-989c-4eed-b294-8690a5007121 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.313740] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199602, 'name': CreateVM_Task, 'duration_secs': 0.291253} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 589.345586] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 589.346452] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 589.346605] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 589.346948] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 589.347675] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-72d4cd2c-6c14-4fb6-acbc-ac087f6efd9a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.350344] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-17b7a14b-10ce-413a-8a1a-df0225c2fa33 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.358957] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3220f606-3b96-4104-ac78-2b6ed659d311 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.362793] env[67752]: DEBUG oslo_vmware.api [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Waiting for the task: (returnval){ [ 589.362793] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52220bcb-5bf0-4b4f-eb14-7f9c0d9faafd" [ 589.362793] env[67752]: _type = "Task" [ 589.362793] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 589.374646] env[67752]: DEBUG nova.compute.provider_tree [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 589.381125] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 589.381278] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 589.381845] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 589.385093] env[67752]: DEBUG nova.scheduler.client.report [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 589.405096] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.355s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 589.405780] env[67752]: DEBUG nova.compute.manager [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 589.464650] env[67752]: DEBUG nova.compute.utils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 589.466266] env[67752]: DEBUG nova.compute.manager [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Not allocating networking since 'none' was specified. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1968}} [ 589.483192] env[67752]: DEBUG nova.compute.manager [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 589.566673] env[67752]: DEBUG nova.compute.manager [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 589.600082] env[67752]: DEBUG nova.virt.hardware [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 589.600537] env[67752]: DEBUG nova.virt.hardware [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 589.600814] env[67752]: DEBUG nova.virt.hardware [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 589.601832] env[67752]: DEBUG nova.virt.hardware [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 589.601832] env[67752]: DEBUG nova.virt.hardware [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 589.601832] env[67752]: DEBUG nova.virt.hardware [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 589.601832] env[67752]: DEBUG nova.virt.hardware [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 589.602091] env[67752]: DEBUG nova.virt.hardware [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 589.602091] env[67752]: DEBUG nova.virt.hardware [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 589.602287] env[67752]: DEBUG nova.virt.hardware [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 589.602483] env[67752]: DEBUG nova.virt.hardware [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 589.603764] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cbf23f1a-f36a-409d-9df1-0de7c523a317 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.613371] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5480b2d0-8c96-4f18-b491-486c52b28e1b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.630018] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Instance VIF info [] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 589.634790] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Creating folder: Project (d323b12d56fa4cc8bd97aa95ac705fed). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 589.635117] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-4fe67cb7-1f03-4e1e-b0da-2dab1c3a8dfd {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.648336] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Created folder: Project (d323b12d56fa4cc8bd97aa95ac705fed) in parent group-v639722. [ 589.648529] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Creating folder: Instances. Parent ref: group-v639732. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 589.648789] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-62017624-64a8-49bd-8132-145c02da0c74 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.657496] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Created folder: Instances in parent group-v639732. [ 589.659312] env[67752]: DEBUG oslo.service.loopingcall [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 589.659312] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 589.659312] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-044d71fd-faa3-4d6e-b4ad-ae8bb7f298ec {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 589.674200] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 589.674200] env[67752]: value = "task-3199605" [ 589.674200] env[67752]: _type = "Task" [ 589.674200] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 589.687971] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199605, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 589.796661] env[67752]: DEBUG nova.policy [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '0becbb0b389d43e9a54ac09dac9046d7', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '27c9a117797543a197520b6fb94ce6fb', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 590.123571] env[67752]: DEBUG nova.network.neutron [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Successfully updated port: 3071faa7-0c8f-4f79-b84f-371bd0685c6d {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 590.160363] env[67752]: DEBUG oslo_concurrency.lockutils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Acquiring lock "refresh_cache-b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 590.160363] env[67752]: DEBUG oslo_concurrency.lockutils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Acquired lock "refresh_cache-b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 590.160363] env[67752]: DEBUG nova.network.neutron [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 590.201205] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199605, 'name': CreateVM_Task} progress is 99%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 590.324668] env[67752]: DEBUG nova.network.neutron [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 590.690181] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199605, 'name': CreateVM_Task} progress is 99%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 591.190925] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199605, 'name': CreateVM_Task, 'duration_secs': 1.387521} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 591.191931] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 591.192170] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 591.192247] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 591.193011] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 591.193011] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-ce461fb4-07af-4bb6-a389-aea687a5b021 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.201131] env[67752]: DEBUG oslo_vmware.api [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Waiting for the task: (returnval){ [ 591.201131] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5264fa58-c6a7-d38e-7f71-0a127fcf59d7" [ 591.201131] env[67752]: _type = "Task" [ 591.201131] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 591.213203] env[67752]: DEBUG oslo_vmware.api [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5264fa58-c6a7-d38e-7f71-0a127fcf59d7, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 591.505280] env[67752]: DEBUG nova.network.neutron [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Updating instance_info_cache with network_info: [{"id": "3071faa7-0c8f-4f79-b84f-371bd0685c6d", "address": "fa:16:3e:6c:8e:0d", "network": {"id": "90a37cf9-29cf-44a5-b057-b0403ea29428", "bridge": "br-int", "label": "tempest-FloatingIPsAssociationNegativeTestJSON-109690407-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "cb995b20e0644427b10975527ced5266", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "c42bb08a-77b4-4bba-8166-702cbb1b5f1e", "external-id": "nsx-vlan-transportzone-137", "segmentation_id": 137, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap3071faa7-0c", "ovs_interfaceid": "3071faa7-0c8f-4f79-b84f-371bd0685c6d", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 591.565789] env[67752]: DEBUG oslo_concurrency.lockutils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Releasing lock "refresh_cache-b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 591.566177] env[67752]: DEBUG nova.compute.manager [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Instance network_info: |[{"id": "3071faa7-0c8f-4f79-b84f-371bd0685c6d", "address": "fa:16:3e:6c:8e:0d", "network": {"id": "90a37cf9-29cf-44a5-b057-b0403ea29428", "bridge": "br-int", "label": "tempest-FloatingIPsAssociationNegativeTestJSON-109690407-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "cb995b20e0644427b10975527ced5266", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "c42bb08a-77b4-4bba-8166-702cbb1b5f1e", "external-id": "nsx-vlan-transportzone-137", "segmentation_id": 137, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap3071faa7-0c", "ovs_interfaceid": "3071faa7-0c8f-4f79-b84f-371bd0685c6d", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 591.567805] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:6c:8e:0d', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'c42bb08a-77b4-4bba-8166-702cbb1b5f1e', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '3071faa7-0c8f-4f79-b84f-371bd0685c6d', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 591.575342] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Creating folder: Project (cb995b20e0644427b10975527ced5266). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 591.576291] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-6c3cae27-a0f4-47a2-a92b-435854a14de7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.587022] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Created folder: Project (cb995b20e0644427b10975527ced5266) in parent group-v639722. [ 591.587868] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Creating folder: Instances. Parent ref: group-v639735. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 591.587868] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-b006c1fe-9a9c-4f31-8bbf-82cc5d3e4358 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.598167] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Created folder: Instances in parent group-v639735. [ 591.598167] env[67752]: DEBUG oslo.service.loopingcall [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 591.598167] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 591.598167] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-5d81f4bc-42a2-4038-89df-455e35286498 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.621598] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 591.621598] env[67752]: value = "task-3199609" [ 591.621598] env[67752]: _type = "Task" [ 591.621598] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 591.637800] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199609, 'name': CreateVM_Task} progress is 6%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 591.652163] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 591.652163] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 591.652244] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 591.652313] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 591.664134] env[67752]: DEBUG nova.compute.manager [req-92f4e5ba-df3d-43cc-81e8-6f2028bf87c1 req-a66208a5-5a2a-4a1b-ac3e-06c65c5dfcf1 service nova] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Received event network-changed-5d7f3c98-5e74-4893-ba90-683172689749 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 591.664488] env[67752]: DEBUG nova.compute.manager [req-92f4e5ba-df3d-43cc-81e8-6f2028bf87c1 req-a66208a5-5a2a-4a1b-ac3e-06c65c5dfcf1 service nova] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Refreshing instance network info cache due to event network-changed-5d7f3c98-5e74-4893-ba90-683172689749. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 591.664488] env[67752]: DEBUG oslo_concurrency.lockutils [req-92f4e5ba-df3d-43cc-81e8-6f2028bf87c1 req-a66208a5-5a2a-4a1b-ac3e-06c65c5dfcf1 service nova] Acquiring lock "refresh_cache-0085fe38-e5ec-44f9-b51b-c54a9f05b60f" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 591.664668] env[67752]: DEBUG oslo_concurrency.lockutils [req-92f4e5ba-df3d-43cc-81e8-6f2028bf87c1 req-a66208a5-5a2a-4a1b-ac3e-06c65c5dfcf1 service nova] Acquired lock "refresh_cache-0085fe38-e5ec-44f9-b51b-c54a9f05b60f" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 591.666328] env[67752]: DEBUG nova.network.neutron [req-92f4e5ba-df3d-43cc-81e8-6f2028bf87c1 req-a66208a5-5a2a-4a1b-ac3e-06c65c5dfcf1 service nova] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Refreshing network info cache for port 5d7f3c98-5e74-4893-ba90-683172689749 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 591.681850] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 591.682491] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 591.683338] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 591.683338] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 591.683338] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 591.683338] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 591.685888] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 591.685978] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 591.686603] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 591.686603] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 591.686603] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 591.686881] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 591.687041] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 591.687227] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 591.709145] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 591.709207] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 591.709356] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 591.709523] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 591.711169] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cbaea8da-bb2c-4cf6-8858-37bd2a0a3938 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.720868] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 591.720868] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 591.721125] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 591.729269] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b6117c97-d5de-40eb-88c6-32314e277882 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.752820] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-601a5ec8-83c9-4966-9c38-854aeebb034b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.763301] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-301785b6-3c64-4061-ac25-3b60f095eeb7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 591.806758] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181008MB free_disk=125GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 591.806938] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 591.807181] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 591.904987] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 0085fe38-e5ec-44f9-b51b-c54a9f05b60f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 591.905163] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 591.905289] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c563b7ed-4fb3-4fb0-8350-789fbe4d53a2 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 591.905410] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fc4da5fa-1921-44e3-9100-042391d4ccb9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 591.905526] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b5eee51b-73b8-4e17-98a4-e108a5b21892 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 591.949690] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fd0cd82b-9339-4415-aa30-af19f46fd3e0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 591.950016] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 5 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 591.950016] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1152MB phys_disk=200GB used_disk=5GB total_vcpus=48 used_vcpus=5 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 592.047517] env[67752]: DEBUG nova.network.neutron [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Successfully created port: e3aaec4f-57ec-4035-80f4-e34fa0bd70d9 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 592.069820] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Acquiring lock "fd0cd82b-9339-4415-aa30-af19f46fd3e0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 592.070076] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Lock "fd0cd82b-9339-4415-aa30-af19f46fd3e0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 592.088258] env[67752]: DEBUG nova.compute.manager [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 592.109034] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aaf9bb68-f57c-42ea-9486-72508e96a8c6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.119586] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-66c2c74a-b6a1-49ae-9132-ca72e322bc5e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.132464] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199609, 'name': CreateVM_Task, 'duration_secs': 0.36578} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 592.158478] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 592.162672] env[67752]: DEBUG oslo_concurrency.lockutils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 592.162840] env[67752]: DEBUG oslo_concurrency.lockutils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 592.164370] env[67752]: DEBUG oslo_concurrency.lockutils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 592.165157] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40206fba-dfd5-4791-b1a3-88b593f7d7d6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.173433] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-fcca60d5-5aa1-4ccb-bed8-e3d3bbe94004 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.182039] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5609ffe7-ee67-458f-9903-1c4a57260eb2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.186601] env[67752]: DEBUG oslo_vmware.api [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Waiting for the task: (returnval){ [ 592.186601] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52e630aa-ad48-1a88-8b83-d177b3b0e31a" [ 592.186601] env[67752]: _type = "Task" [ 592.186601] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 592.188274] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 592.199191] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 592.205749] env[67752]: DEBUG oslo_concurrency.lockutils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 592.206023] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 592.206249] env[67752]: DEBUG oslo_concurrency.lockutils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 592.238309] env[67752]: ERROR nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [req-09eec1e1-da6c-4386-a63c-f201b44dee31] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID c7141b98-bb6c-46bd-b37f-a8e913940199. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-09eec1e1-da6c-4386-a63c-f201b44dee31"}]} [ 592.265376] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Refreshing inventories for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:804}} [ 592.288021] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Updating ProviderTree inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:768}} [ 592.288021] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 592.305548] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Refreshing aggregate associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, aggregates: None {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:813}} [ 592.336975] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Refreshing trait associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, traits: COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:825}} [ 592.463710] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ef0df1a-dfdc-4c95-a2ab-b2f44ceda9ea {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.472576] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a3309957-1a6e-4eba-b572-c9177fa8a1fd {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.505953] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-197529e7-8234-4ca6-b79b-85eaf61de3f8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.516342] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-882e23a3-425f-4a62-a800-abf1c1ecc59a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.532607] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 592.600021] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Updated inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with generation 10 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:957}} [ 592.600021] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Updating resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 generation from 10 to 11 during operation: update_inventory {{(pid=67752) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 592.600021] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 592.633820] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 592.633820] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.826s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 592.633820] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.446s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 592.638754] env[67752]: INFO nova.compute.claims [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 592.868466] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2fc24242-5b72-4e62-a330-f6a7e463711c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.877377] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-02b51841-08c3-41a5-88e4-596e815d5c1a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.911197] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bb5f8aa6-c9f0-4279-b23e-429760d7f629 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.919474] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-058836b9-9adf-43ac-99dc-8eb288af2970 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 592.935900] env[67752]: DEBUG nova.compute.provider_tree [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 592.948703] env[67752]: DEBUG nova.scheduler.client.report [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 592.969922] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.336s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 592.970489] env[67752]: DEBUG nova.compute.manager [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 593.021134] env[67752]: DEBUG nova.compute.utils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 593.022712] env[67752]: DEBUG nova.compute.manager [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 593.022800] env[67752]: DEBUG nova.network.neutron [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 593.044013] env[67752]: DEBUG nova.compute.manager [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 593.066338] env[67752]: DEBUG nova.network.neutron [req-92f4e5ba-df3d-43cc-81e8-6f2028bf87c1 req-a66208a5-5a2a-4a1b-ac3e-06c65c5dfcf1 service nova] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Updated VIF entry in instance network info cache for port 5d7f3c98-5e74-4893-ba90-683172689749. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 593.066690] env[67752]: DEBUG nova.network.neutron [req-92f4e5ba-df3d-43cc-81e8-6f2028bf87c1 req-a66208a5-5a2a-4a1b-ac3e-06c65c5dfcf1 service nova] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Updating instance_info_cache with network_info: [{"id": "5d7f3c98-5e74-4893-ba90-683172689749", "address": "fa:16:3e:46:85:5e", "network": {"id": "8754f810-4855-45bb-9a57-6c1511f0722f", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.180", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "c43e0a2618b34312898e0e4bb014b17c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "983826cf-6390-4ec6-bf97-30a1060947fc", "external-id": "nsx-vlan-transportzone-367", "segmentation_id": 367, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5d7f3c98-5e", "ovs_interfaceid": "5d7f3c98-5e74-4893-ba90-683172689749", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 593.080695] env[67752]: DEBUG oslo_concurrency.lockutils [req-92f4e5ba-df3d-43cc-81e8-6f2028bf87c1 req-a66208a5-5a2a-4a1b-ac3e-06c65c5dfcf1 service nova] Releasing lock "refresh_cache-0085fe38-e5ec-44f9-b51b-c54a9f05b60f" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 593.145972] env[67752]: DEBUG nova.compute.manager [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 593.174753] env[67752]: DEBUG nova.virt.hardware [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 593.175017] env[67752]: DEBUG nova.virt.hardware [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 593.175180] env[67752]: DEBUG nova.virt.hardware [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 593.175479] env[67752]: DEBUG nova.virt.hardware [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 593.175570] env[67752]: DEBUG nova.virt.hardware [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 593.175647] env[67752]: DEBUG nova.virt.hardware [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 593.176063] env[67752]: DEBUG nova.virt.hardware [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 593.176247] env[67752]: DEBUG nova.virt.hardware [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 593.176580] env[67752]: DEBUG nova.virt.hardware [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 593.176667] env[67752]: DEBUG nova.virt.hardware [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 593.177036] env[67752]: DEBUG nova.virt.hardware [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 593.177776] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e199b1f3-4be4-42ba-b7d4-afb442336bd4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 593.186876] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97f459ab-986e-4ddf-964e-168d7a4c84f5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 593.304340] env[67752]: DEBUG nova.policy [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '8bab4698b7b844109955b5442518391b', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '43513b7c3e4349d292ef8eea922e7bbf', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 595.176318] env[67752]: DEBUG nova.network.neutron [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Successfully created port: ac8a4102-0252-4d6e-9011-282f6235e08c {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 595.398323] env[67752]: DEBUG nova.network.neutron [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Successfully updated port: e3aaec4f-57ec-4035-80f4-e34fa0bd70d9 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 595.410172] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Acquiring lock "refresh_cache-fc4da5fa-1921-44e3-9100-042391d4ccb9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 595.410172] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Acquired lock "refresh_cache-fc4da5fa-1921-44e3-9100-042391d4ccb9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 595.410172] env[67752]: DEBUG nova.network.neutron [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 595.558308] env[67752]: DEBUG nova.network.neutron [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 596.637687] env[67752]: DEBUG nova.network.neutron [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Updating instance_info_cache with network_info: [{"id": "e3aaec4f-57ec-4035-80f4-e34fa0bd70d9", "address": "fa:16:3e:66:fc:eb", "network": {"id": "8754f810-4855-45bb-9a57-6c1511f0722f", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.109", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "c43e0a2618b34312898e0e4bb014b17c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "983826cf-6390-4ec6-bf97-30a1060947fc", "external-id": "nsx-vlan-transportzone-367", "segmentation_id": 367, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape3aaec4f-57", "ovs_interfaceid": "e3aaec4f-57ec-4035-80f4-e34fa0bd70d9", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 596.657940] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Releasing lock "refresh_cache-fc4da5fa-1921-44e3-9100-042391d4ccb9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 596.657940] env[67752]: DEBUG nova.compute.manager [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Instance network_info: |[{"id": "e3aaec4f-57ec-4035-80f4-e34fa0bd70d9", "address": "fa:16:3e:66:fc:eb", "network": {"id": "8754f810-4855-45bb-9a57-6c1511f0722f", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.109", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "c43e0a2618b34312898e0e4bb014b17c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "983826cf-6390-4ec6-bf97-30a1060947fc", "external-id": "nsx-vlan-transportzone-367", "segmentation_id": 367, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape3aaec4f-57", "ovs_interfaceid": "e3aaec4f-57ec-4035-80f4-e34fa0bd70d9", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 596.658180] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:66:fc:eb', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '983826cf-6390-4ec6-bf97-30a1060947fc', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'e3aaec4f-57ec-4035-80f4-e34fa0bd70d9', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 596.665978] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Creating folder: Project (27c9a117797543a197520b6fb94ce6fb). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 596.666730] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-950c2b79-10fc-45d3-b079-22057a935941 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 596.679558] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Created folder: Project (27c9a117797543a197520b6fb94ce6fb) in parent group-v639722. [ 596.680268] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Creating folder: Instances. Parent ref: group-v639738. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 596.680605] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-0fcb986c-76b0-4f60-b0f6-98301056dda7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 596.692194] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Created folder: Instances in parent group-v639738. [ 596.692436] env[67752]: DEBUG oslo.service.loopingcall [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 596.693151] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 596.693151] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-562b8150-60e1-4337-b02c-76e40a865e64 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 596.719622] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 596.719622] env[67752]: value = "task-3199614" [ 596.719622] env[67752]: _type = "Task" [ 596.719622] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 596.730033] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199614, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 597.232657] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199614, 'name': CreateVM_Task, 'duration_secs': 0.344007} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 597.233184] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 597.235073] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 597.236345] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 597.236345] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 597.237252] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-e5a8ebcb-3889-496b-8b66-8404f24c0388 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 597.243689] env[67752]: DEBUG oslo_vmware.api [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Waiting for the task: (returnval){ [ 597.243689] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]522d3d50-0876-5450-3682-750625893d92" [ 597.243689] env[67752]: _type = "Task" [ 597.243689] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 597.255364] env[67752]: DEBUG oslo_vmware.api [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]522d3d50-0876-5450-3682-750625893d92, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 597.552367] env[67752]: DEBUG nova.compute.manager [req-5f4ae442-a32d-471a-a93d-31db080cf83b req-dc3f653c-f563-4989-aec1-4681bd22dfdc service nova] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Received event network-vif-plugged-3071faa7-0c8f-4f79-b84f-371bd0685c6d {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 597.552367] env[67752]: DEBUG oslo_concurrency.lockutils [req-5f4ae442-a32d-471a-a93d-31db080cf83b req-dc3f653c-f563-4989-aec1-4681bd22dfdc service nova] Acquiring lock "b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 597.552367] env[67752]: DEBUG oslo_concurrency.lockutils [req-5f4ae442-a32d-471a-a93d-31db080cf83b req-dc3f653c-f563-4989-aec1-4681bd22dfdc service nova] Lock "b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 597.552367] env[67752]: DEBUG oslo_concurrency.lockutils [req-5f4ae442-a32d-471a-a93d-31db080cf83b req-dc3f653c-f563-4989-aec1-4681bd22dfdc service nova] Lock "b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 597.553492] env[67752]: DEBUG nova.compute.manager [req-5f4ae442-a32d-471a-a93d-31db080cf83b req-dc3f653c-f563-4989-aec1-4681bd22dfdc service nova] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] No waiting events found dispatching network-vif-plugged-3071faa7-0c8f-4f79-b84f-371bd0685c6d {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 597.553897] env[67752]: WARNING nova.compute.manager [req-5f4ae442-a32d-471a-a93d-31db080cf83b req-dc3f653c-f563-4989-aec1-4681bd22dfdc service nova] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Received unexpected event network-vif-plugged-3071faa7-0c8f-4f79-b84f-371bd0685c6d for instance with vm_state building and task_state spawning. [ 597.554281] env[67752]: DEBUG nova.compute.manager [req-5f4ae442-a32d-471a-a93d-31db080cf83b req-dc3f653c-f563-4989-aec1-4681bd22dfdc service nova] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Received event network-changed-3071faa7-0c8f-4f79-b84f-371bd0685c6d {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 597.554568] env[67752]: DEBUG nova.compute.manager [req-5f4ae442-a32d-471a-a93d-31db080cf83b req-dc3f653c-f563-4989-aec1-4681bd22dfdc service nova] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Refreshing instance network info cache due to event network-changed-3071faa7-0c8f-4f79-b84f-371bd0685c6d. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 597.554884] env[67752]: DEBUG oslo_concurrency.lockutils [req-5f4ae442-a32d-471a-a93d-31db080cf83b req-dc3f653c-f563-4989-aec1-4681bd22dfdc service nova] Acquiring lock "refresh_cache-b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 597.555135] env[67752]: DEBUG oslo_concurrency.lockutils [req-5f4ae442-a32d-471a-a93d-31db080cf83b req-dc3f653c-f563-4989-aec1-4681bd22dfdc service nova] Acquired lock "refresh_cache-b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 597.555841] env[67752]: DEBUG nova.network.neutron [req-5f4ae442-a32d-471a-a93d-31db080cf83b req-dc3f653c-f563-4989-aec1-4681bd22dfdc service nova] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Refreshing network info cache for port 3071faa7-0c8f-4f79-b84f-371bd0685c6d {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 597.755646] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 597.755977] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 597.756127] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 599.067515] env[67752]: DEBUG nova.network.neutron [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Successfully updated port: ac8a4102-0252-4d6e-9011-282f6235e08c {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 599.082761] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Acquiring lock "refresh_cache-fd0cd82b-9339-4415-aa30-af19f46fd3e0" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 599.083289] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Acquired lock "refresh_cache-fd0cd82b-9339-4415-aa30-af19f46fd3e0" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 599.084334] env[67752]: DEBUG nova.network.neutron [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 599.142931] env[67752]: DEBUG nova.network.neutron [req-5f4ae442-a32d-471a-a93d-31db080cf83b req-dc3f653c-f563-4989-aec1-4681bd22dfdc service nova] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Updated VIF entry in instance network info cache for port 3071faa7-0c8f-4f79-b84f-371bd0685c6d. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 599.143384] env[67752]: DEBUG nova.network.neutron [req-5f4ae442-a32d-471a-a93d-31db080cf83b req-dc3f653c-f563-4989-aec1-4681bd22dfdc service nova] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Updating instance_info_cache with network_info: [{"id": "3071faa7-0c8f-4f79-b84f-371bd0685c6d", "address": "fa:16:3e:6c:8e:0d", "network": {"id": "90a37cf9-29cf-44a5-b057-b0403ea29428", "bridge": "br-int", "label": "tempest-FloatingIPsAssociationNegativeTestJSON-109690407-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "cb995b20e0644427b10975527ced5266", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "c42bb08a-77b4-4bba-8166-702cbb1b5f1e", "external-id": "nsx-vlan-transportzone-137", "segmentation_id": 137, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap3071faa7-0c", "ovs_interfaceid": "3071faa7-0c8f-4f79-b84f-371bd0685c6d", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 599.162903] env[67752]: DEBUG oslo_concurrency.lockutils [req-5f4ae442-a32d-471a-a93d-31db080cf83b req-dc3f653c-f563-4989-aec1-4681bd22dfdc service nova] Releasing lock "refresh_cache-b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 599.218085] env[67752]: DEBUG nova.network.neutron [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 599.857999] env[67752]: DEBUG nova.network.neutron [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Updating instance_info_cache with network_info: [{"id": "ac8a4102-0252-4d6e-9011-282f6235e08c", "address": "fa:16:3e:98:6f:3b", "network": {"id": "27c64865-1594-49cd-8088-449cfa545168", "bridge": "br-int", "label": "tempest-ImagesOneServerNegativeTestJSON-1366422682-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "43513b7c3e4349d292ef8eea922e7bbf", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "0467beaa-08c6-44d6-b8a2-e9c609c21ff4", "external-id": "nsx-vlan-transportzone-540", "segmentation_id": 540, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapac8a4102-02", "ovs_interfaceid": "ac8a4102-0252-4d6e-9011-282f6235e08c", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 599.873431] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Releasing lock "refresh_cache-fd0cd82b-9339-4415-aa30-af19f46fd3e0" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 599.873748] env[67752]: DEBUG nova.compute.manager [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Instance network_info: |[{"id": "ac8a4102-0252-4d6e-9011-282f6235e08c", "address": "fa:16:3e:98:6f:3b", "network": {"id": "27c64865-1594-49cd-8088-449cfa545168", "bridge": "br-int", "label": "tempest-ImagesOneServerNegativeTestJSON-1366422682-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "43513b7c3e4349d292ef8eea922e7bbf", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "0467beaa-08c6-44d6-b8a2-e9c609c21ff4", "external-id": "nsx-vlan-transportzone-540", "segmentation_id": 540, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapac8a4102-02", "ovs_interfaceid": "ac8a4102-0252-4d6e-9011-282f6235e08c", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 599.874421] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:98:6f:3b', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '0467beaa-08c6-44d6-b8a2-e9c609c21ff4', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'ac8a4102-0252-4d6e-9011-282f6235e08c', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 599.883059] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Creating folder: Project (43513b7c3e4349d292ef8eea922e7bbf). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 599.883059] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-ca928214-55c7-48a3-aac7-bdf66667937d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 599.896076] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Created folder: Project (43513b7c3e4349d292ef8eea922e7bbf) in parent group-v639722. [ 599.896326] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Creating folder: Instances. Parent ref: group-v639742. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 599.896577] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-5d38dbb3-b385-4db3-9aea-b2d73e292f17 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 599.913040] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Created folder: Instances in parent group-v639742. [ 599.915240] env[67752]: DEBUG oslo.service.loopingcall [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 599.915539] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 599.915788] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-ca663ab3-cd3e-4e22-a274-eebfa399b5a3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 599.942625] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 599.942625] env[67752]: value = "task-3199619" [ 599.942625] env[67752]: _type = "Task" [ 599.942625] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 599.954950] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199619, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 600.456438] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199619, 'name': CreateVM_Task, 'duration_secs': 0.314641} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 600.456668] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 600.458625] env[67752]: DEBUG oslo_vmware.service [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-18d6c7e0-d005-4497-a897-25182fb989f4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 600.469950] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 600.469950] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Acquired lock "[datastore1] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 600.469950] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Acquired external semaphore "[datastore1] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 600.470130] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-9e80ac79-5381-4b63-8be8-2f881044ad63 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 600.478713] env[67752]: DEBUG oslo_vmware.api [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Waiting for the task: (returnval){ [ 600.478713] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5212d479-2c47-49be-4948-ad0df3baacf9" [ 600.478713] env[67752]: _type = "Task" [ 600.478713] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 600.493551] env[67752]: DEBUG oslo_vmware.api [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5212d479-2c47-49be-4948-ad0df3baacf9, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 600.783629] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Acquiring lock "39479042-b891-4f25-9992-47312c6b6b43" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 600.784414] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Lock "39479042-b891-4f25-9992-47312c6b6b43" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 600.803610] env[67752]: DEBUG nova.compute.manager [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 600.864387] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 600.864670] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 600.866221] env[67752]: INFO nova.compute.claims [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 600.996044] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Releasing lock "[datastore1] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 600.996188] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 600.996410] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Acquiring lock "[datastore1] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 600.996558] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Acquired lock "[datastore1] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 600.996717] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Creating directory with path [datastore1] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 600.997191] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-c63e2608-2727-4ecb-a017-d8f59cb911f9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.009483] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Created directory with path [datastore1] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 601.009667] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Folder [datastore1] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 601.010535] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4cf212a-60cc-4e4c-9c61-ff476cd7e74f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.027039] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-a6ba0fe3-e5bc-4368-9e1c-752fdef97194 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.038632] env[67752]: DEBUG oslo_vmware.api [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Waiting for the task: (returnval){ [ 601.038632] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5236b22e-dcbe-802d-d279-9e03401c24ab" [ 601.038632] env[67752]: _type = "Task" [ 601.038632] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 601.051027] env[67752]: DEBUG oslo_vmware.api [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5236b22e-dcbe-802d-d279-9e03401c24ab, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 601.134937] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2972aa74-7283-43f2-9c36-ce274f7c6420 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.147698] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e797fa4-6a79-4cfa-a9d8-a4db8ee2e6a6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.188731] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40154da0-e39a-430b-8b43-8c5b3646f131 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.197696] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08ad03f0-bc21-46d4-b4f7-3b15c1d923ba {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.214424] env[67752]: DEBUG nova.compute.provider_tree [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 601.236631] env[67752]: DEBUG nova.scheduler.client.report [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 601.262165] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.397s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 601.262395] env[67752]: DEBUG nova.compute.manager [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 601.311116] env[67752]: DEBUG nova.compute.utils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 601.315444] env[67752]: DEBUG nova.compute.manager [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 601.315444] env[67752]: DEBUG nova.network.neutron [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 601.341548] env[67752]: DEBUG nova.compute.manager [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 601.449851] env[67752]: DEBUG nova.compute.manager [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 601.488217] env[67752]: DEBUG nova.virt.hardware [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 601.488538] env[67752]: DEBUG nova.virt.hardware [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 601.488648] env[67752]: DEBUG nova.virt.hardware [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 601.488864] env[67752]: DEBUG nova.virt.hardware [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 601.488974] env[67752]: DEBUG nova.virt.hardware [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 601.489255] env[67752]: DEBUG nova.virt.hardware [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 601.489536] env[67752]: DEBUG nova.virt.hardware [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 601.489725] env[67752]: DEBUG nova.virt.hardware [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 601.489943] env[67752]: DEBUG nova.virt.hardware [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 601.490283] env[67752]: DEBUG nova.virt.hardware [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 601.490346] env[67752]: DEBUG nova.virt.hardware [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 601.491600] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-04c96547-3645-44d9-bf01-30d71d825045 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.496838] env[67752]: DEBUG nova.policy [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '4dde095c4bb244b98d5c19ab8d8cf0b2', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '4f55c933274d4cd992aa7a5ff0055d6d', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 601.506776] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3655003a-47df-489e-a64e-4f70127e5a4a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.560277] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 601.560575] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Creating directory with path [datastore1] vmware_temp/12554a4c-c719-440f-b42e-de8c370be4d6/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 601.560893] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-5177ee01-b60d-4860-80e8-12f31a098750 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.591206] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Created directory with path [datastore1] vmware_temp/12554a4c-c719-440f-b42e-de8c370be4d6/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 601.591206] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Fetch image to [datastore1] vmware_temp/12554a4c-c719-440f-b42e-de8c370be4d6/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 601.591483] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore1] vmware_temp/12554a4c-c719-440f-b42e-de8c370be4d6/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore1 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 601.594134] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f86031bd-5696-44d6-8eed-b851a2f59163 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.607336] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e40132be-4957-4859-9ac8-e8f9d3c1be81 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.623327] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db675adf-4f5d-43d8-9eaf-46fb19812318 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.665420] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-57b7ac79-316a-46fd-9477-37ae84cb8eba {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.670972] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-b7fe01c8-7378-4918-bdaa-ae08e629dfa7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 601.771191] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore1 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 601.860351] env[67752]: DEBUG oslo_vmware.rw_handles [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/12554a4c-c719-440f-b42e-de8c370be4d6/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 601.935611] env[67752]: DEBUG nova.compute.manager [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Received event network-vif-plugged-e3aaec4f-57ec-4035-80f4-e34fa0bd70d9 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 601.936179] env[67752]: DEBUG oslo_concurrency.lockutils [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] Acquiring lock "fc4da5fa-1921-44e3-9100-042391d4ccb9-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 601.936542] env[67752]: DEBUG oslo_concurrency.lockutils [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] Lock "fc4da5fa-1921-44e3-9100-042391d4ccb9-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 601.936922] env[67752]: DEBUG oslo_concurrency.lockutils [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] Lock "fc4da5fa-1921-44e3-9100-042391d4ccb9-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 601.937172] env[67752]: DEBUG nova.compute.manager [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] No waiting events found dispatching network-vif-plugged-e3aaec4f-57ec-4035-80f4-e34fa0bd70d9 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 601.938265] env[67752]: WARNING nova.compute.manager [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Received unexpected event network-vif-plugged-e3aaec4f-57ec-4035-80f4-e34fa0bd70d9 for instance with vm_state building and task_state spawning. [ 601.938265] env[67752]: DEBUG nova.compute.manager [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Received event network-changed-e3aaec4f-57ec-4035-80f4-e34fa0bd70d9 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 601.938265] env[67752]: DEBUG nova.compute.manager [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Refreshing instance network info cache due to event network-changed-e3aaec4f-57ec-4035-80f4-e34fa0bd70d9. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 601.938265] env[67752]: DEBUG oslo_concurrency.lockutils [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] Acquiring lock "refresh_cache-fc4da5fa-1921-44e3-9100-042391d4ccb9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 601.938265] env[67752]: DEBUG oslo_concurrency.lockutils [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] Acquired lock "refresh_cache-fc4da5fa-1921-44e3-9100-042391d4ccb9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 601.938707] env[67752]: DEBUG nova.network.neutron [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Refreshing network info cache for port e3aaec4f-57ec-4035-80f4-e34fa0bd70d9 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 601.947483] env[67752]: DEBUG oslo_vmware.rw_handles [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 601.947764] env[67752]: DEBUG oslo_vmware.rw_handles [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Closing write handle for https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/12554a4c-c719-440f-b42e-de8c370be4d6/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore1. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 602.541757] env[67752]: DEBUG nova.network.neutron [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Successfully created port: b77ad48d-58bd-424a-8181-0d6abe7495e0 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 602.972086] env[67752]: DEBUG nova.network.neutron [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Updated VIF entry in instance network info cache for port e3aaec4f-57ec-4035-80f4-e34fa0bd70d9. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 602.972376] env[67752]: DEBUG nova.network.neutron [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Updating instance_info_cache with network_info: [{"id": "e3aaec4f-57ec-4035-80f4-e34fa0bd70d9", "address": "fa:16:3e:66:fc:eb", "network": {"id": "8754f810-4855-45bb-9a57-6c1511f0722f", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.109", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "c43e0a2618b34312898e0e4bb014b17c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "983826cf-6390-4ec6-bf97-30a1060947fc", "external-id": "nsx-vlan-transportzone-367", "segmentation_id": 367, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape3aaec4f-57", "ovs_interfaceid": "e3aaec4f-57ec-4035-80f4-e34fa0bd70d9", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 602.990853] env[67752]: DEBUG oslo_concurrency.lockutils [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] Releasing lock "refresh_cache-fc4da5fa-1921-44e3-9100-042391d4ccb9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 602.991411] env[67752]: DEBUG nova.compute.manager [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Received event network-vif-plugged-ac8a4102-0252-4d6e-9011-282f6235e08c {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 602.991411] env[67752]: DEBUG oslo_concurrency.lockutils [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] Acquiring lock "fd0cd82b-9339-4415-aa30-af19f46fd3e0-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 602.991585] env[67752]: DEBUG oslo_concurrency.lockutils [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] Lock "fd0cd82b-9339-4415-aa30-af19f46fd3e0-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 602.991949] env[67752]: DEBUG oslo_concurrency.lockutils [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] Lock "fd0cd82b-9339-4415-aa30-af19f46fd3e0-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 602.991949] env[67752]: DEBUG nova.compute.manager [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] No waiting events found dispatching network-vif-plugged-ac8a4102-0252-4d6e-9011-282f6235e08c {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 602.992088] env[67752]: WARNING nova.compute.manager [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Received unexpected event network-vif-plugged-ac8a4102-0252-4d6e-9011-282f6235e08c for instance with vm_state building and task_state spawning. [ 602.992476] env[67752]: DEBUG nova.compute.manager [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Received event network-changed-ac8a4102-0252-4d6e-9011-282f6235e08c {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 602.992476] env[67752]: DEBUG nova.compute.manager [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Refreshing instance network info cache due to event network-changed-ac8a4102-0252-4d6e-9011-282f6235e08c. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 602.992733] env[67752]: DEBUG oslo_concurrency.lockutils [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] Acquiring lock "refresh_cache-fd0cd82b-9339-4415-aa30-af19f46fd3e0" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 602.992788] env[67752]: DEBUG oslo_concurrency.lockutils [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] Acquired lock "refresh_cache-fd0cd82b-9339-4415-aa30-af19f46fd3e0" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 602.992899] env[67752]: DEBUG nova.network.neutron [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Refreshing network info cache for port ac8a4102-0252-4d6e-9011-282f6235e08c {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 603.802105] env[67752]: DEBUG nova.network.neutron [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Updated VIF entry in instance network info cache for port ac8a4102-0252-4d6e-9011-282f6235e08c. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 603.802519] env[67752]: DEBUG nova.network.neutron [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Updating instance_info_cache with network_info: [{"id": "ac8a4102-0252-4d6e-9011-282f6235e08c", "address": "fa:16:3e:98:6f:3b", "network": {"id": "27c64865-1594-49cd-8088-449cfa545168", "bridge": "br-int", "label": "tempest-ImagesOneServerNegativeTestJSON-1366422682-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "43513b7c3e4349d292ef8eea922e7bbf", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "0467beaa-08c6-44d6-b8a2-e9c609c21ff4", "external-id": "nsx-vlan-transportzone-540", "segmentation_id": 540, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapac8a4102-02", "ovs_interfaceid": "ac8a4102-0252-4d6e-9011-282f6235e08c", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 603.813785] env[67752]: DEBUG oslo_concurrency.lockutils [req-d93e2870-8335-4e04-b349-bb29936e09ea req-47e969e7-9d4a-40eb-bff8-fd9e0a0b2130 service nova] Releasing lock "refresh_cache-fd0cd82b-9339-4415-aa30-af19f46fd3e0" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 604.429125] env[67752]: DEBUG nova.network.neutron [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Successfully updated port: b77ad48d-58bd-424a-8181-0d6abe7495e0 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 604.443714] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Acquiring lock "refresh_cache-39479042-b891-4f25-9992-47312c6b6b43" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 604.443845] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Acquired lock "refresh_cache-39479042-b891-4f25-9992-47312c6b6b43" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 604.444484] env[67752]: DEBUG nova.network.neutron [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 604.503185] env[67752]: DEBUG nova.network.neutron [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 604.513898] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquiring lock "9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 604.514202] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 604.535837] env[67752]: DEBUG nova.compute.manager [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 604.611012] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 604.611012] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 604.612486] env[67752]: INFO nova.compute.claims [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 604.817049] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d9d4898-833f-4dc0-bed1-28128498a9b2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 604.825305] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8656e30-24b1-406d-9a44-05e58e0f0b26 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 604.833505] env[67752]: DEBUG nova.network.neutron [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Updating instance_info_cache with network_info: [{"id": "b77ad48d-58bd-424a-8181-0d6abe7495e0", "address": "fa:16:3e:0a:25:18", "network": {"id": "1ee158f6-ffa2-45dd-8603-5761a0a51b80", "bridge": "br-int", "label": "tempest-ImagesOneServerTestJSON-1990467959-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "4f55c933274d4cd992aa7a5ff0055d6d", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f01bbee7-8b9a-46be-891e-59d8142fb359", "external-id": "nsx-vlan-transportzone-145", "segmentation_id": 145, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapb77ad48d-58", "ovs_interfaceid": "b77ad48d-58bd-424a-8181-0d6abe7495e0", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 604.862699] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ec30343-8801-43a6-836a-2929c6da6597 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 604.865602] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Releasing lock "refresh_cache-39479042-b891-4f25-9992-47312c6b6b43" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 604.865862] env[67752]: DEBUG nova.compute.manager [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Instance network_info: |[{"id": "b77ad48d-58bd-424a-8181-0d6abe7495e0", "address": "fa:16:3e:0a:25:18", "network": {"id": "1ee158f6-ffa2-45dd-8603-5761a0a51b80", "bridge": "br-int", "label": "tempest-ImagesOneServerTestJSON-1990467959-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "4f55c933274d4cd992aa7a5ff0055d6d", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f01bbee7-8b9a-46be-891e-59d8142fb359", "external-id": "nsx-vlan-transportzone-145", "segmentation_id": 145, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapb77ad48d-58", "ovs_interfaceid": "b77ad48d-58bd-424a-8181-0d6abe7495e0", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 604.866550] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:0a:25:18', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'f01bbee7-8b9a-46be-891e-59d8142fb359', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'b77ad48d-58bd-424a-8181-0d6abe7495e0', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 604.874584] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Creating folder: Project (4f55c933274d4cd992aa7a5ff0055d6d). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 604.875561] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-517160da-a9a6-4a02-b36d-de32f12f6e31 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 604.881718] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c398e5f4-c88a-450a-a4ac-07d6ce983e8f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 604.895557] env[67752]: DEBUG nova.compute.provider_tree [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 604.897759] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Created folder: Project (4f55c933274d4cd992aa7a5ff0055d6d) in parent group-v639722. [ 604.897941] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Creating folder: Instances. Parent ref: group-v639745. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 604.898386] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-1f03c9bb-8800-456a-b8cd-5f8bda51f32e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 604.907943] env[67752]: DEBUG nova.scheduler.client.report [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 604.914790] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Created folder: Instances in parent group-v639745. [ 604.914790] env[67752]: DEBUG oslo.service.loopingcall [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 604.914790] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 604.914790] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-d06db44c-559e-48e0-9ef0-38a780552702 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 604.928760] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.318s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 604.929255] env[67752]: DEBUG nova.compute.manager [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 604.942265] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 604.942265] env[67752]: value = "task-3199624" [ 604.942265] env[67752]: _type = "Task" [ 604.942265] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 604.954213] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199624, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 605.013112] env[67752]: DEBUG nova.compute.utils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 605.014855] env[67752]: DEBUG nova.compute.manager [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 605.015380] env[67752]: DEBUG nova.network.neutron [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 605.028418] env[67752]: DEBUG nova.compute.manager [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 605.143070] env[67752]: DEBUG nova.compute.manager [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 605.163225] env[67752]: DEBUG nova.policy [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '33981823d3944c1ea4b8dd3bc816b58f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '623ecf0022e946a7ae2c085680601380', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 605.177101] env[67752]: DEBUG nova.virt.hardware [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 605.177935] env[67752]: DEBUG nova.virt.hardware [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 605.177935] env[67752]: DEBUG nova.virt.hardware [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 605.177935] env[67752]: DEBUG nova.virt.hardware [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 605.178082] env[67752]: DEBUG nova.virt.hardware [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 605.178221] env[67752]: DEBUG nova.virt.hardware [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 605.178484] env[67752]: DEBUG nova.virt.hardware [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 605.178790] env[67752]: DEBUG nova.virt.hardware [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 605.178885] env[67752]: DEBUG nova.virt.hardware [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 605.182027] env[67752]: DEBUG nova.virt.hardware [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 605.182027] env[67752]: DEBUG nova.virt.hardware [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 605.182027] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab6bc090-1e04-43e2-a6b2-8983df1db7b6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.190828] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6497acfa-8bec-4129-836f-fe1980b5e094 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.302244] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquiring lock "5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 605.302457] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 605.319581] env[67752]: DEBUG nova.compute.manager [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 605.382777] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 605.383101] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 605.384585] env[67752]: INFO nova.compute.claims [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 605.454793] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199624, 'name': CreateVM_Task, 'duration_secs': 0.376165} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 605.454793] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 605.456964] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 605.457344] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 605.457678] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 605.457966] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-57f93afb-97ac-4189-8fb2-92289a0c60a8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.466179] env[67752]: DEBUG oslo_vmware.api [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Waiting for the task: (returnval){ [ 605.466179] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5285b39d-97d5-c324-d0ce-c16d288db105" [ 605.466179] env[67752]: _type = "Task" [ 605.466179] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 605.480702] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 605.481103] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 605.481362] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 605.599556] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c1ab666a-d328-4e3f-9371-cbd3c2c8b97c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.608556] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4c7c382c-4840-47e7-a044-55e608f75f35 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.651153] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2c6a14b2-db0e-4a53-a040-5a90659386c9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.657361] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45476564-2b8c-4312-849e-c6f1429a275c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.671597] env[67752]: DEBUG nova.compute.provider_tree [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 605.683569] env[67752]: DEBUG nova.scheduler.client.report [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 605.702594] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.319s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 605.703536] env[67752]: DEBUG nova.compute.manager [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 605.750846] env[67752]: DEBUG nova.compute.utils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 605.752601] env[67752]: DEBUG nova.compute.manager [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 605.752814] env[67752]: DEBUG nova.network.neutron [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 605.764604] env[67752]: DEBUG nova.compute.manager [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 605.842199] env[67752]: DEBUG nova.compute.manager [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 605.870600] env[67752]: DEBUG nova.virt.hardware [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 605.870859] env[67752]: DEBUG nova.virt.hardware [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 605.871073] env[67752]: DEBUG nova.virt.hardware [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 605.871556] env[67752]: DEBUG nova.virt.hardware [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 605.872378] env[67752]: DEBUG nova.virt.hardware [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 605.873036] env[67752]: DEBUG nova.virt.hardware [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 605.873036] env[67752]: DEBUG nova.virt.hardware [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 605.873036] env[67752]: DEBUG nova.virt.hardware [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 605.873251] env[67752]: DEBUG nova.virt.hardware [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 605.873744] env[67752]: DEBUG nova.virt.hardware [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 605.873744] env[67752]: DEBUG nova.virt.hardware [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 605.874553] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-746f3420-24d5-466c-9c5d-68c35a1b66dc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 605.887376] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f7f9fa1e-e727-4575-9235-432f42c72c63 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 606.152741] env[67752]: DEBUG nova.policy [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '39f478bef41a4a92bc28a8df8f7bd404', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '217e25d875894435842a418b92ff6960', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 606.180712] env[67752]: DEBUG nova.network.neutron [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Successfully created port: a3bbde71-df35-471a-958b-dce5bbc197c0 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 606.244721] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquiring lock "b0cc1acd-0e33-4547-a43b-549b3de81728" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 606.244954] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "b0cc1acd-0e33-4547-a43b-549b3de81728" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 606.256879] env[67752]: DEBUG nova.compute.manager [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 606.304944] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 606.305226] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 606.307152] env[67752]: INFO nova.compute.claims [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 606.496876] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09fe2816-6450-48d9-bac6-10b0ab1730b1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 606.505692] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d4c6db0-b598-4c29-a4f3-f22dd2ed88cd {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 606.536759] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-41014fc8-a0b2-4b78-863c-81bc5513bb3a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 606.544959] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2f72846-c80a-4db5-bd14-d5c395d6040d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 606.558767] env[67752]: DEBUG nova.compute.provider_tree [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 606.571529] env[67752]: DEBUG nova.scheduler.client.report [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 606.593794] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.288s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 606.594368] env[67752]: DEBUG nova.compute.manager [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 606.629688] env[67752]: DEBUG nova.compute.utils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 606.634019] env[67752]: DEBUG nova.compute.manager [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 606.634019] env[67752]: DEBUG nova.network.neutron [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 606.639665] env[67752]: DEBUG nova.compute.manager [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 606.702818] env[67752]: DEBUG nova.compute.manager [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 606.727973] env[67752]: DEBUG nova.virt.hardware [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 606.728268] env[67752]: DEBUG nova.virt.hardware [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 606.728474] env[67752]: DEBUG nova.virt.hardware [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 606.729032] env[67752]: DEBUG nova.virt.hardware [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 606.729218] env[67752]: DEBUG nova.virt.hardware [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 606.729376] env[67752]: DEBUG nova.virt.hardware [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 606.729589] env[67752]: DEBUG nova.virt.hardware [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 606.729754] env[67752]: DEBUG nova.virt.hardware [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 606.729988] env[67752]: DEBUG nova.virt.hardware [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 606.730181] env[67752]: DEBUG nova.virt.hardware [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 606.730360] env[67752]: DEBUG nova.virt.hardware [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 606.731227] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-11db51db-9375-4767-856f-6d111f21c696 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 606.740168] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-493e349d-dab3-41b1-9ee2-56982a3c6c01 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 606.972526] env[67752]: DEBUG nova.policy [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '6c1e3aad166a4d5c8de9c632b2ab2139', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3a36f872b5c044a5a0dcbce1e454fcd9', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 606.979460] env[67752]: DEBUG nova.compute.manager [req-e94ac2ae-dd73-48de-b0d8-c90e8e5800aa req-6100d218-a3aa-4a63-8066-63fdb6b13384 service nova] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Received event network-vif-plugged-b77ad48d-58bd-424a-8181-0d6abe7495e0 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 606.979701] env[67752]: DEBUG oslo_concurrency.lockutils [req-e94ac2ae-dd73-48de-b0d8-c90e8e5800aa req-6100d218-a3aa-4a63-8066-63fdb6b13384 service nova] Acquiring lock "39479042-b891-4f25-9992-47312c6b6b43-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 606.980067] env[67752]: DEBUG oslo_concurrency.lockutils [req-e94ac2ae-dd73-48de-b0d8-c90e8e5800aa req-6100d218-a3aa-4a63-8066-63fdb6b13384 service nova] Lock "39479042-b891-4f25-9992-47312c6b6b43-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 606.980403] env[67752]: DEBUG oslo_concurrency.lockutils [req-e94ac2ae-dd73-48de-b0d8-c90e8e5800aa req-6100d218-a3aa-4a63-8066-63fdb6b13384 service nova] Lock "39479042-b891-4f25-9992-47312c6b6b43-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 606.980625] env[67752]: DEBUG nova.compute.manager [req-e94ac2ae-dd73-48de-b0d8-c90e8e5800aa req-6100d218-a3aa-4a63-8066-63fdb6b13384 service nova] [instance: 39479042-b891-4f25-9992-47312c6b6b43] No waiting events found dispatching network-vif-plugged-b77ad48d-58bd-424a-8181-0d6abe7495e0 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 606.980975] env[67752]: WARNING nova.compute.manager [req-e94ac2ae-dd73-48de-b0d8-c90e8e5800aa req-6100d218-a3aa-4a63-8066-63fdb6b13384 service nova] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Received unexpected event network-vif-plugged-b77ad48d-58bd-424a-8181-0d6abe7495e0 for instance with vm_state building and task_state spawning. [ 607.410211] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Acquiring lock "eca7c94f-9bd1-4c15-988e-d4e33086ba40" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 607.410211] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Lock "eca7c94f-9bd1-4c15-988e-d4e33086ba40" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 607.770674] env[67752]: DEBUG nova.network.neutron [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Successfully created port: 3ecae4a1-6beb-4167-93e4-7cd030c2aba2 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 608.662909] env[67752]: DEBUG nova.network.neutron [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Successfully updated port: a3bbde71-df35-471a-958b-dce5bbc197c0 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 608.679317] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquiring lock "refresh_cache-9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 608.679317] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquired lock "refresh_cache-9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 608.679317] env[67752]: DEBUG nova.network.neutron [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 608.682953] env[67752]: DEBUG nova.network.neutron [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Successfully created port: 5eff2e54-5109-47b3-87af-46e7aec36b9d {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 609.003656] env[67752]: DEBUG nova.network.neutron [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 609.730528] env[67752]: DEBUG nova.network.neutron [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Updating instance_info_cache with network_info: [{"id": "a3bbde71-df35-471a-958b-dce5bbc197c0", "address": "fa:16:3e:c4:ac:43", "network": {"id": "17c092d8-0dd2-45ff-8347-48693bf6460a", "bridge": "br-int", "label": "tempest-VolumesAdminNegativeTest-733494480-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.11", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "623ecf0022e946a7ae2c085680601380", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f2c424c9-6446-4b2a-af8c-4d9c29117c39", "external-id": "nsx-vlan-transportzone-437", "segmentation_id": 437, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapa3bbde71-df", "ovs_interfaceid": "a3bbde71-df35-471a-958b-dce5bbc197c0", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 609.744272] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Releasing lock "refresh_cache-9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 609.744575] env[67752]: DEBUG nova.compute.manager [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Instance network_info: |[{"id": "a3bbde71-df35-471a-958b-dce5bbc197c0", "address": "fa:16:3e:c4:ac:43", "network": {"id": "17c092d8-0dd2-45ff-8347-48693bf6460a", "bridge": "br-int", "label": "tempest-VolumesAdminNegativeTest-733494480-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.11", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "623ecf0022e946a7ae2c085680601380", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f2c424c9-6446-4b2a-af8c-4d9c29117c39", "external-id": "nsx-vlan-transportzone-437", "segmentation_id": 437, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapa3bbde71-df", "ovs_interfaceid": "a3bbde71-df35-471a-958b-dce5bbc197c0", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 609.745035] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:c4:ac:43', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'f2c424c9-6446-4b2a-af8c-4d9c29117c39', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'a3bbde71-df35-471a-958b-dce5bbc197c0', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 609.754097] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Creating folder: Project (623ecf0022e946a7ae2c085680601380). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 609.754685] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-d5ae55f9-378d-42be-b5fb-9149f10f87b3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 609.766892] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Created folder: Project (623ecf0022e946a7ae2c085680601380) in parent group-v639722. [ 609.767113] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Creating folder: Instances. Parent ref: group-v639748. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 609.767347] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-e60e9250-bba7-4f7f-b927-79c38d42be61 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 609.784017] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Created folder: Instances in parent group-v639748. [ 609.784314] env[67752]: DEBUG oslo.service.loopingcall [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 609.784865] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 609.785124] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-e99380ef-07b8-47f8-85db-9a7fa70613f4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 609.811078] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 609.811078] env[67752]: value = "task-3199627" [ 609.811078] env[67752]: _type = "Task" [ 609.811078] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 609.820402] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199627, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 609.833391] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquiring lock "ff4895f8-0374-4b34-a5e8-d3ba24f840a5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 609.833608] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "ff4895f8-0374-4b34-a5e8-d3ba24f840a5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 610.321669] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199627, 'name': CreateVM_Task, 'duration_secs': 0.308638} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 610.321845] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 610.323239] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 610.323239] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 610.323239] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 610.323428] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-21620f85-2fdb-4ef9-85c2-c1d9d90503e2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 610.332175] env[67752]: DEBUG oslo_vmware.api [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Waiting for the task: (returnval){ [ 610.332175] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52735285-040d-e014-1cfc-558e56d7fb73" [ 610.332175] env[67752]: _type = "Task" [ 610.332175] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 610.342598] env[67752]: DEBUG oslo_vmware.api [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52735285-040d-e014-1cfc-558e56d7fb73, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 610.545270] env[67752]: DEBUG nova.network.neutron [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Successfully updated port: 3ecae4a1-6beb-4167-93e4-7cd030c2aba2 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 610.557461] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquiring lock "refresh_cache-5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 610.557461] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquired lock "refresh_cache-5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 610.557581] env[67752]: DEBUG nova.network.neutron [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 610.661035] env[67752]: DEBUG nova.network.neutron [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 610.844311] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 610.844311] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 610.844311] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 611.102278] env[67752]: DEBUG nova.network.neutron [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Updating instance_info_cache with network_info: [{"id": "3ecae4a1-6beb-4167-93e4-7cd030c2aba2", "address": "fa:16:3e:13:9c:7f", "network": {"id": "8754f810-4855-45bb-9a57-6c1511f0722f", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.164", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "c43e0a2618b34312898e0e4bb014b17c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "983826cf-6390-4ec6-bf97-30a1060947fc", "external-id": "nsx-vlan-transportzone-367", "segmentation_id": 367, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap3ecae4a1-6b", "ovs_interfaceid": "3ecae4a1-6beb-4167-93e4-7cd030c2aba2", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 611.120274] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Releasing lock "refresh_cache-5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 611.120322] env[67752]: DEBUG nova.compute.manager [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Instance network_info: |[{"id": "3ecae4a1-6beb-4167-93e4-7cd030c2aba2", "address": "fa:16:3e:13:9c:7f", "network": {"id": "8754f810-4855-45bb-9a57-6c1511f0722f", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.164", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "c43e0a2618b34312898e0e4bb014b17c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "983826cf-6390-4ec6-bf97-30a1060947fc", "external-id": "nsx-vlan-transportzone-367", "segmentation_id": 367, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap3ecae4a1-6b", "ovs_interfaceid": "3ecae4a1-6beb-4167-93e4-7cd030c2aba2", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 611.121698] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:13:9c:7f', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '983826cf-6390-4ec6-bf97-30a1060947fc', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '3ecae4a1-6beb-4167-93e4-7cd030c2aba2', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 611.133149] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Creating folder: Project (217e25d875894435842a418b92ff6960). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 611.134440] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-e79d1277-7b2b-4970-a870-33dbe786079d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 611.140791] env[67752]: DEBUG oslo_concurrency.lockutils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Acquiring lock "ca18af6b-6b80-44f3-bd8b-f61e2596acd7" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 611.140942] env[67752]: DEBUG oslo_concurrency.lockutils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Lock "ca18af6b-6b80-44f3-bd8b-f61e2596acd7" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 611.146791] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Created folder: Project (217e25d875894435842a418b92ff6960) in parent group-v639722. [ 611.146919] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Creating folder: Instances. Parent ref: group-v639751. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 611.147640] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-6e32b9fc-90d4-4f78-95fd-0f76458bcc5f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 611.158682] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Created folder: Instances in parent group-v639751. [ 611.159037] env[67752]: DEBUG oslo.service.loopingcall [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 611.159624] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 611.159781] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-d04e2b74-79ad-4e10-a50e-9e2afdb6c284 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 611.182538] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 611.182538] env[67752]: value = "task-3199630" [ 611.182538] env[67752]: _type = "Task" [ 611.182538] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 611.195426] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199630, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 611.212439] env[67752]: DEBUG nova.network.neutron [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Successfully updated port: 5eff2e54-5109-47b3-87af-46e7aec36b9d {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 611.228684] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Acquiring lock "cbe3fe72-5a91-4c81-ac62-be6592aa806a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 611.229213] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Lock "cbe3fe72-5a91-4c81-ac62-be6592aa806a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 611.232776] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquiring lock "refresh_cache-b0cc1acd-0e33-4547-a43b-549b3de81728" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 611.232776] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquired lock "refresh_cache-b0cc1acd-0e33-4547-a43b-549b3de81728" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 611.232776] env[67752]: DEBUG nova.network.neutron [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 611.333992] env[67752]: DEBUG nova.network.neutron [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 611.695033] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199630, 'name': CreateVM_Task, 'duration_secs': 0.332575} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 611.695033] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 611.699365] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 611.699365] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 611.699365] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 611.699466] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-5bc44ca2-22d5-4fa4-8573-afb5400c3b82 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 611.705592] env[67752]: DEBUG oslo_vmware.api [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Waiting for the task: (returnval){ [ 611.705592] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5222714c-60cc-d420-5cef-0fd8d7231e77" [ 611.705592] env[67752]: _type = "Task" [ 611.705592] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 611.717441] env[67752]: DEBUG oslo_vmware.api [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5222714c-60cc-d420-5cef-0fd8d7231e77, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 611.929359] env[67752]: DEBUG nova.network.neutron [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Updating instance_info_cache with network_info: [{"id": "5eff2e54-5109-47b3-87af-46e7aec36b9d", "address": "fa:16:3e:46:13:7e", "network": {"id": "8754f810-4855-45bb-9a57-6c1511f0722f", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.158", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "c43e0a2618b34312898e0e4bb014b17c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "983826cf-6390-4ec6-bf97-30a1060947fc", "external-id": "nsx-vlan-transportzone-367", "segmentation_id": 367, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5eff2e54-51", "ovs_interfaceid": "5eff2e54-5109-47b3-87af-46e7aec36b9d", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 611.948280] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Releasing lock "refresh_cache-b0cc1acd-0e33-4547-a43b-549b3de81728" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 611.948596] env[67752]: DEBUG nova.compute.manager [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Instance network_info: |[{"id": "5eff2e54-5109-47b3-87af-46e7aec36b9d", "address": "fa:16:3e:46:13:7e", "network": {"id": "8754f810-4855-45bb-9a57-6c1511f0722f", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.158", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "c43e0a2618b34312898e0e4bb014b17c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "983826cf-6390-4ec6-bf97-30a1060947fc", "external-id": "nsx-vlan-transportzone-367", "segmentation_id": 367, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5eff2e54-51", "ovs_interfaceid": "5eff2e54-5109-47b3-87af-46e7aec36b9d", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 611.949131] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:46:13:7e', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '983826cf-6390-4ec6-bf97-30a1060947fc', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '5eff2e54-5109-47b3-87af-46e7aec36b9d', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 611.962963] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Creating folder: Project (3a36f872b5c044a5a0dcbce1e454fcd9). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 611.962963] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-25563768-2c21-4015-a8bf-898374ed3fa3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 611.974046] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Created folder: Project (3a36f872b5c044a5a0dcbce1e454fcd9) in parent group-v639722. [ 611.974260] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Creating folder: Instances. Parent ref: group-v639754. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 611.976164] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-603c453e-67e3-40c5-97fb-010b0e285220 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 611.989879] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Created folder: Instances in parent group-v639754. [ 611.990208] env[67752]: DEBUG oslo.service.loopingcall [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 611.990429] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 611.990639] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-d15c4477-b5c8-4811-b70a-9ea54d3862de {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 612.016794] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 612.016794] env[67752]: value = "task-3199633" [ 612.016794] env[67752]: _type = "Task" [ 612.016794] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 612.025257] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199633, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 612.218680] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 612.218994] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 612.219689] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 612.529683] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199633, 'name': CreateVM_Task, 'duration_secs': 0.309878} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 612.529897] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 612.530635] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 612.530705] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 612.531038] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 612.531264] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-f1178a77-1cc2-4d16-92e5-585603afd286 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 612.536457] env[67752]: DEBUG oslo_vmware.api [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Waiting for the task: (returnval){ [ 612.536457] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]529b8d1c-8c9c-7ad0-1eee-1f81c60180ce" [ 612.536457] env[67752]: _type = "Task" [ 612.536457] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 612.548158] env[67752]: DEBUG oslo_vmware.api [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]529b8d1c-8c9c-7ad0-1eee-1f81c60180ce, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 612.899174] env[67752]: DEBUG nova.compute.manager [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Received event network-changed-b77ad48d-58bd-424a-8181-0d6abe7495e0 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 612.899370] env[67752]: DEBUG nova.compute.manager [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Refreshing instance network info cache due to event network-changed-b77ad48d-58bd-424a-8181-0d6abe7495e0. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 612.899581] env[67752]: DEBUG oslo_concurrency.lockutils [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] Acquiring lock "refresh_cache-39479042-b891-4f25-9992-47312c6b6b43" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 612.899723] env[67752]: DEBUG oslo_concurrency.lockutils [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] Acquired lock "refresh_cache-39479042-b891-4f25-9992-47312c6b6b43" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 612.899916] env[67752]: DEBUG nova.network.neutron [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Refreshing network info cache for port b77ad48d-58bd-424a-8181-0d6abe7495e0 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 613.050111] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 613.050429] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 613.050834] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 613.431714] env[67752]: DEBUG nova.network.neutron [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Updated VIF entry in instance network info cache for port b77ad48d-58bd-424a-8181-0d6abe7495e0. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 613.432170] env[67752]: DEBUG nova.network.neutron [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Updating instance_info_cache with network_info: [{"id": "b77ad48d-58bd-424a-8181-0d6abe7495e0", "address": "fa:16:3e:0a:25:18", "network": {"id": "1ee158f6-ffa2-45dd-8603-5761a0a51b80", "bridge": "br-int", "label": "tempest-ImagesOneServerTestJSON-1990467959-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "4f55c933274d4cd992aa7a5ff0055d6d", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f01bbee7-8b9a-46be-891e-59d8142fb359", "external-id": "nsx-vlan-transportzone-145", "segmentation_id": 145, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapb77ad48d-58", "ovs_interfaceid": "b77ad48d-58bd-424a-8181-0d6abe7495e0", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 613.443046] env[67752]: DEBUG oslo_concurrency.lockutils [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] Releasing lock "refresh_cache-39479042-b891-4f25-9992-47312c6b6b43" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 613.443301] env[67752]: DEBUG nova.compute.manager [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Received event network-vif-plugged-a3bbde71-df35-471a-958b-dce5bbc197c0 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 613.443494] env[67752]: DEBUG oslo_concurrency.lockutils [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] Acquiring lock "9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.443704] env[67752]: DEBUG oslo_concurrency.lockutils [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] Lock "9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.443856] env[67752]: DEBUG oslo_concurrency.lockutils [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] Lock "9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.444032] env[67752]: DEBUG nova.compute.manager [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] No waiting events found dispatching network-vif-plugged-a3bbde71-df35-471a-958b-dce5bbc197c0 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 613.444206] env[67752]: WARNING nova.compute.manager [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Received unexpected event network-vif-plugged-a3bbde71-df35-471a-958b-dce5bbc197c0 for instance with vm_state building and task_state spawning. [ 613.444364] env[67752]: DEBUG nova.compute.manager [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Received event network-changed-a3bbde71-df35-471a-958b-dce5bbc197c0 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 613.444517] env[67752]: DEBUG nova.compute.manager [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Refreshing instance network info cache due to event network-changed-a3bbde71-df35-471a-958b-dce5bbc197c0. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 613.444773] env[67752]: DEBUG oslo_concurrency.lockutils [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] Acquiring lock "refresh_cache-9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 613.444854] env[67752]: DEBUG oslo_concurrency.lockutils [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] Acquired lock "refresh_cache-9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 613.445263] env[67752]: DEBUG nova.network.neutron [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Refreshing network info cache for port a3bbde71-df35-471a-958b-dce5bbc197c0 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 614.150906] env[67752]: DEBUG nova.network.neutron [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Updated VIF entry in instance network info cache for port a3bbde71-df35-471a-958b-dce5bbc197c0. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 614.151373] env[67752]: DEBUG nova.network.neutron [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Updating instance_info_cache with network_info: [{"id": "a3bbde71-df35-471a-958b-dce5bbc197c0", "address": "fa:16:3e:c4:ac:43", "network": {"id": "17c092d8-0dd2-45ff-8347-48693bf6460a", "bridge": "br-int", "label": "tempest-VolumesAdminNegativeTest-733494480-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.11", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "623ecf0022e946a7ae2c085680601380", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f2c424c9-6446-4b2a-af8c-4d9c29117c39", "external-id": "nsx-vlan-transportzone-437", "segmentation_id": 437, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapa3bbde71-df", "ovs_interfaceid": "a3bbde71-df35-471a-958b-dce5bbc197c0", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 614.160717] env[67752]: DEBUG oslo_concurrency.lockutils [req-87ccc976-beb0-4283-aee6-4ea1b50ebabe req-842bb68d-47ee-4a7f-b3e5-cd45534c1b4f service nova] Releasing lock "refresh_cache-9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 616.393496] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Acquiring lock "02942b29-2343-441e-9e2b-aca2af57c849" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 616.393751] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Lock "02942b29-2343-441e-9e2b-aca2af57c849" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 616.440273] env[67752]: DEBUG nova.compute.manager [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Received event network-vif-plugged-3ecae4a1-6beb-4167-93e4-7cd030c2aba2 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 616.440327] env[67752]: DEBUG oslo_concurrency.lockutils [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] Acquiring lock "5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 616.441234] env[67752]: DEBUG oslo_concurrency.lockutils [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] Lock "5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 616.441234] env[67752]: DEBUG oslo_concurrency.lockutils [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] Lock "5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 616.441234] env[67752]: DEBUG nova.compute.manager [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] No waiting events found dispatching network-vif-plugged-3ecae4a1-6beb-4167-93e4-7cd030c2aba2 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 616.441234] env[67752]: WARNING nova.compute.manager [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Received unexpected event network-vif-plugged-3ecae4a1-6beb-4167-93e4-7cd030c2aba2 for instance with vm_state building and task_state spawning. [ 616.441528] env[67752]: DEBUG nova.compute.manager [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Received event network-changed-3ecae4a1-6beb-4167-93e4-7cd030c2aba2 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 616.441528] env[67752]: DEBUG nova.compute.manager [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Refreshing instance network info cache due to event network-changed-3ecae4a1-6beb-4167-93e4-7cd030c2aba2. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 616.441638] env[67752]: DEBUG oslo_concurrency.lockutils [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] Acquiring lock "refresh_cache-5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 616.441726] env[67752]: DEBUG oslo_concurrency.lockutils [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] Acquired lock "refresh_cache-5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 616.441881] env[67752]: DEBUG nova.network.neutron [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Refreshing network info cache for port 3ecae4a1-6beb-4167-93e4-7cd030c2aba2 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 617.097448] env[67752]: DEBUG nova.network.neutron [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Updated VIF entry in instance network info cache for port 3ecae4a1-6beb-4167-93e4-7cd030c2aba2. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 617.097819] env[67752]: DEBUG nova.network.neutron [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Updating instance_info_cache with network_info: [{"id": "3ecae4a1-6beb-4167-93e4-7cd030c2aba2", "address": "fa:16:3e:13:9c:7f", "network": {"id": "8754f810-4855-45bb-9a57-6c1511f0722f", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.164", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "c43e0a2618b34312898e0e4bb014b17c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "983826cf-6390-4ec6-bf97-30a1060947fc", "external-id": "nsx-vlan-transportzone-367", "segmentation_id": 367, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap3ecae4a1-6b", "ovs_interfaceid": "3ecae4a1-6beb-4167-93e4-7cd030c2aba2", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 617.113743] env[67752]: DEBUG oslo_concurrency.lockutils [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] Releasing lock "refresh_cache-5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 617.113960] env[67752]: DEBUG nova.compute.manager [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Received event network-vif-plugged-5eff2e54-5109-47b3-87af-46e7aec36b9d {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 617.114182] env[67752]: DEBUG oslo_concurrency.lockutils [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] Acquiring lock "b0cc1acd-0e33-4547-a43b-549b3de81728-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 617.114430] env[67752]: DEBUG oslo_concurrency.lockutils [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] Lock "b0cc1acd-0e33-4547-a43b-549b3de81728-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 617.114605] env[67752]: DEBUG oslo_concurrency.lockutils [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] Lock "b0cc1acd-0e33-4547-a43b-549b3de81728-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 617.114771] env[67752]: DEBUG nova.compute.manager [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] No waiting events found dispatching network-vif-plugged-5eff2e54-5109-47b3-87af-46e7aec36b9d {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 617.114940] env[67752]: WARNING nova.compute.manager [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Received unexpected event network-vif-plugged-5eff2e54-5109-47b3-87af-46e7aec36b9d for instance with vm_state building and task_state spawning. [ 617.115119] env[67752]: DEBUG nova.compute.manager [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Received event network-changed-5eff2e54-5109-47b3-87af-46e7aec36b9d {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 617.115296] env[67752]: DEBUG nova.compute.manager [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Refreshing instance network info cache due to event network-changed-5eff2e54-5109-47b3-87af-46e7aec36b9d. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 617.115489] env[67752]: DEBUG oslo_concurrency.lockutils [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] Acquiring lock "refresh_cache-b0cc1acd-0e33-4547-a43b-549b3de81728" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 617.115628] env[67752]: DEBUG oslo_concurrency.lockutils [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] Acquired lock "refresh_cache-b0cc1acd-0e33-4547-a43b-549b3de81728" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 617.115840] env[67752]: DEBUG nova.network.neutron [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Refreshing network info cache for port 5eff2e54-5109-47b3-87af-46e7aec36b9d {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 617.679347] env[67752]: DEBUG nova.network.neutron [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Updated VIF entry in instance network info cache for port 5eff2e54-5109-47b3-87af-46e7aec36b9d. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 617.679732] env[67752]: DEBUG nova.network.neutron [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Updating instance_info_cache with network_info: [{"id": "5eff2e54-5109-47b3-87af-46e7aec36b9d", "address": "fa:16:3e:46:13:7e", "network": {"id": "8754f810-4855-45bb-9a57-6c1511f0722f", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.158", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "c43e0a2618b34312898e0e4bb014b17c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "983826cf-6390-4ec6-bf97-30a1060947fc", "external-id": "nsx-vlan-transportzone-367", "segmentation_id": 367, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5eff2e54-51", "ovs_interfaceid": "5eff2e54-5109-47b3-87af-46e7aec36b9d", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 617.694698] env[67752]: DEBUG oslo_concurrency.lockutils [req-c0e6f93b-eb07-4d73-a599-3a17dc406ba9 req-69a2c596-dfaf-43fb-903d-1cf6f0f6bf28 service nova] Releasing lock "refresh_cache-b0cc1acd-0e33-4547-a43b-549b3de81728" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 622.657188] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Acquiring lock "ebd6c719-32ae-410c-a444-db7ad3f8a9c5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 622.657508] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Lock "ebd6c719-32ae-410c-a444-db7ad3f8a9c5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 624.888791] env[67752]: DEBUG oslo_concurrency.lockutils [None req-06ef1eed-1b6d-4546-8b73-0988ea7f027c tempest-ServerRescueTestJSON-960884533 tempest-ServerRescueTestJSON-960884533-project-member] Acquiring lock "f528dc15-1a86-4af5-ac45-536fe3dfe101" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 624.889131] env[67752]: DEBUG oslo_concurrency.lockutils [None req-06ef1eed-1b6d-4546-8b73-0988ea7f027c tempest-ServerRescueTestJSON-960884533 tempest-ServerRescueTestJSON-960884533-project-member] Lock "f528dc15-1a86-4af5-ac45-536fe3dfe101" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 631.301664] env[67752]: DEBUG oslo_concurrency.lockutils [None req-13c59bfe-ce4d-4478-876f-40c71e3a478f tempest-ServerDiagnosticsTest-1513525778 tempest-ServerDiagnosticsTest-1513525778-project-member] Acquiring lock "4c0873a7-b8e5-4d3e-bdc0-3d236e3b9df6" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 631.301664] env[67752]: DEBUG oslo_concurrency.lockutils [None req-13c59bfe-ce4d-4478-876f-40c71e3a478f tempest-ServerDiagnosticsTest-1513525778 tempest-ServerDiagnosticsTest-1513525778-project-member] Lock "4c0873a7-b8e5-4d3e-bdc0-3d236e3b9df6" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 632.296268] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4e50a843-ce43-49a2-960c-709c4cf9a01e tempest-ServersV294TestFqdnHostnames-724274089 tempest-ServersV294TestFqdnHostnames-724274089-project-member] Acquiring lock "e3c2e1cf-cae5-474f-9f61-96d79815d846" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 632.298360] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4e50a843-ce43-49a2-960c-709c4cf9a01e tempest-ServersV294TestFqdnHostnames-724274089 tempest-ServersV294TestFqdnHostnames-724274089-project-member] Lock "e3c2e1cf-cae5-474f-9f61-96d79815d846" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 633.951636] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b45c275f-58d0-4291-a4d3-b0aa404c21cf tempest-ServerActionsTestOtherB-2033325558 tempest-ServerActionsTestOtherB-2033325558-project-member] Acquiring lock "f22bcbae-0dcb-4666-b5e7-00c560e016c3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 633.951988] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b45c275f-58d0-4291-a4d3-b0aa404c21cf tempest-ServerActionsTestOtherB-2033325558 tempest-ServerActionsTestOtherB-2033325558-project-member] Lock "f22bcbae-0dcb-4666-b5e7-00c560e016c3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 634.117142] env[67752]: WARNING oslo_vmware.rw_handles [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 634.117142] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 634.117142] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 634.117142] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 634.117142] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 634.117142] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 634.117142] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 634.117142] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 634.117142] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 634.117142] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 634.117142] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 634.117142] env[67752]: ERROR oslo_vmware.rw_handles [ 634.117142] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/b43599fd-9897-4c19-b277-723581846f65/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 634.117737] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 634.117737] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Copying Virtual Disk [datastore2] vmware_temp/b43599fd-9897-4c19-b277-723581846f65/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/b43599fd-9897-4c19-b277-723581846f65/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 634.117737] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-6d37a594-3acc-491c-b7e6-e83e02676dbe {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 634.129915] env[67752]: DEBUG oslo_vmware.api [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Waiting for the task: (returnval){ [ 634.129915] env[67752]: value = "task-3199634" [ 634.129915] env[67752]: _type = "Task" [ 634.129915] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 634.139650] env[67752]: DEBUG oslo_vmware.api [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Task: {'id': task-3199634, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 634.646037] env[67752]: DEBUG oslo_vmware.exceptions [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 634.650507] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 634.656930] env[67752]: ERROR nova.compute.manager [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 634.656930] env[67752]: Faults: ['InvalidArgument'] [ 634.656930] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Traceback (most recent call last): [ 634.656930] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 634.656930] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] yield resources [ 634.656930] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 634.656930] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] self.driver.spawn(context, instance, image_meta, [ 634.656930] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 634.656930] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 634.656930] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 634.656930] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] self._fetch_image_if_missing(context, vi) [ 634.656930] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 634.657390] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] image_cache(vi, tmp_image_ds_loc) [ 634.657390] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 634.657390] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] vm_util.copy_virtual_disk( [ 634.657390] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 634.657390] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] session._wait_for_task(vmdk_copy_task) [ 634.657390] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 634.657390] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] return self.wait_for_task(task_ref) [ 634.657390] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 634.657390] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] return evt.wait() [ 634.657390] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 634.657390] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] result = hub.switch() [ 634.657390] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 634.657390] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] return self.greenlet.switch() [ 634.657755] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 634.657755] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] self.f(*self.args, **self.kw) [ 634.657755] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 634.657755] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] raise exceptions.translate_fault(task_info.error) [ 634.657755] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 634.657755] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Faults: ['InvalidArgument'] [ 634.657755] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] [ 634.657755] env[67752]: INFO nova.compute.manager [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Terminating instance [ 634.657755] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 634.658068] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 634.659852] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-ad6b10e8-183d-4c3b-aaf1-53f829d9845e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 634.661933] env[67752]: DEBUG nova.compute.manager [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 634.662310] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 634.662990] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a398e90-3808-43c7-bf76-3d10c27a897f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 634.673698] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 634.675177] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-d1071cc6-8c51-4939-9e6f-444a42f8caa1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 634.676978] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 634.676978] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 634.677538] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-61f09683-a7a7-4883-8550-4f40e0b0153b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 634.683934] env[67752]: DEBUG oslo_vmware.api [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Waiting for the task: (returnval){ [ 634.683934] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52ec0cf1-0654-005f-918d-b0889f52f345" [ 634.683934] env[67752]: _type = "Task" [ 634.683934] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 634.694176] env[67752]: DEBUG oslo_vmware.api [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52ec0cf1-0654-005f-918d-b0889f52f345, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 634.748149] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 634.748543] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 634.748757] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Deleting the datastore file [datastore2] 0085fe38-e5ec-44f9-b51b-c54a9f05b60f {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 634.749044] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-291c21e4-dffb-4d79-8467-9bd246cedf1c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 634.757281] env[67752]: DEBUG oslo_vmware.api [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Waiting for the task: (returnval){ [ 634.757281] env[67752]: value = "task-3199636" [ 634.757281] env[67752]: _type = "Task" [ 634.757281] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 634.769723] env[67752]: DEBUG oslo_vmware.api [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Task: {'id': task-3199636, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 635.198019] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 635.198019] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Creating directory with path [datastore2] vmware_temp/9b796c90-1b4f-4a7f-8778-a0c0267d2797/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 635.198019] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-16e15ce3-9635-407c-beb0-061a47a14ac1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.211348] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Created directory with path [datastore2] vmware_temp/9b796c90-1b4f-4a7f-8778-a0c0267d2797/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 635.211348] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Fetch image to [datastore2] vmware_temp/9b796c90-1b4f-4a7f-8778-a0c0267d2797/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 635.211560] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/9b796c90-1b4f-4a7f-8778-a0c0267d2797/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 635.213550] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54f3dfa2-3b28-4b80-851c-1f7ebaf0d6c2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.220989] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-88076e32-db3d-4a3b-b236-4d354be1d05b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.230520] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-452feec2-912a-4b73-9a9e-1a09e6621104 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.267222] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-964e46ea-1aa9-48be-a54d-3cfab02ddf50 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.276025] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-36f84a65-6add-4f68-ab89-644a605ece6f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.277981] env[67752]: DEBUG oslo_vmware.api [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Task: {'id': task-3199636, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.069941} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 635.278302] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 635.278414] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 635.278581] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 635.280885] env[67752]: INFO nova.compute.manager [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Took 0.62 seconds to destroy the instance on the hypervisor. [ 635.283166] env[67752]: DEBUG nova.compute.claims [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 635.283355] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 635.283580] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 635.303234] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 635.377659] env[67752]: DEBUG oslo_vmware.rw_handles [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/9b796c90-1b4f-4a7f-8778-a0c0267d2797/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 635.450573] env[67752]: DEBUG oslo_vmware.rw_handles [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 635.450776] env[67752]: DEBUG oslo_vmware.rw_handles [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Closing write handle for https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/9b796c90-1b4f-4a7f-8778-a0c0267d2797/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 635.757054] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a5dd47e5-7242-4077-9e2e-6ae032981935 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.765200] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8451d94-b2f0-42a1-9e88-1d58732b8ff6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.796710] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d0377532-e8e0-4169-a3ea-fbf3c3d528be {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.804213] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2eb39ea2-6dd2-469f-ba56-33557f484c5d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 635.817422] env[67752]: DEBUG nova.compute.provider_tree [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 635.854477] env[67752]: ERROR nova.scheduler.client.report [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [req-a83e883e-5a5f-4bd0-8a5a-2e1df8b46439] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID c7141b98-bb6c-46bd-b37f-a8e913940199. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-a83e883e-5a5f-4bd0-8a5a-2e1df8b46439"}]}: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 635.875040] env[67752]: DEBUG nova.scheduler.client.report [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Refreshing inventories for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:804}} [ 635.895673] env[67752]: DEBUG nova.scheduler.client.report [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Updating ProviderTree inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:768}} [ 635.896115] env[67752]: DEBUG nova.compute.provider_tree [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 635.915923] env[67752]: DEBUG nova.scheduler.client.report [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Refreshing aggregate associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, aggregates: None {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:813}} [ 635.936303] env[67752]: DEBUG nova.scheduler.client.report [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Refreshing trait associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, traits: COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:825}} [ 636.264790] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-977e4311-865b-4321-8ab8-e166a50a10f1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 636.274353] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-49be790f-321b-4ed3-b592-ce0f69752146 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 636.314300] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cc966867-0a3c-4dff-92ff-249ad86c7e8b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 636.323668] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ce57d62c-7fd1-47da-92b2-ca71eac5f8cb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 636.339431] env[67752]: DEBUG nova.compute.provider_tree [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 636.385476] env[67752]: DEBUG nova.scheduler.client.report [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Updated inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with generation 25 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:957}} [ 636.385767] env[67752]: DEBUG nova.compute.provider_tree [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Updating resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 generation from 25 to 26 during operation: update_inventory {{(pid=67752) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 636.386053] env[67752]: DEBUG nova.compute.provider_tree [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 636.415067] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.131s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 636.415298] env[67752]: ERROR nova.compute.manager [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 636.415298] env[67752]: Faults: ['InvalidArgument'] [ 636.415298] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Traceback (most recent call last): [ 636.415298] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 636.415298] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] self.driver.spawn(context, instance, image_meta, [ 636.415298] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 636.415298] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 636.415298] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 636.415298] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] self._fetch_image_if_missing(context, vi) [ 636.415298] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 636.415298] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] image_cache(vi, tmp_image_ds_loc) [ 636.415298] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 636.415639] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] vm_util.copy_virtual_disk( [ 636.415639] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 636.415639] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] session._wait_for_task(vmdk_copy_task) [ 636.415639] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 636.415639] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] return self.wait_for_task(task_ref) [ 636.415639] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 636.415639] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] return evt.wait() [ 636.415639] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 636.415639] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] result = hub.switch() [ 636.415639] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 636.415639] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] return self.greenlet.switch() [ 636.415639] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 636.415639] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] self.f(*self.args, **self.kw) [ 636.415981] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 636.415981] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] raise exceptions.translate_fault(task_info.error) [ 636.415981] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 636.415981] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Faults: ['InvalidArgument'] [ 636.415981] env[67752]: ERROR nova.compute.manager [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] [ 636.416135] env[67752]: DEBUG nova.compute.utils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 636.422267] env[67752]: DEBUG nova.compute.manager [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Build of instance 0085fe38-e5ec-44f9-b51b-c54a9f05b60f was re-scheduled: A specified parameter was not correct: fileType [ 636.422267] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 636.422267] env[67752]: DEBUG nova.compute.manager [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 636.422267] env[67752]: DEBUG nova.compute.manager [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 636.423386] env[67752]: DEBUG nova.compute.manager [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 636.423386] env[67752]: DEBUG nova.network.neutron [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 636.651937] env[67752]: DEBUG oslo_concurrency.lockutils [None req-29de0487-2264-4436-8ac3-d6e9fd25d33f tempest-ServerTagsTestJSON-923615097 tempest-ServerTagsTestJSON-923615097-project-member] Acquiring lock "c38268b5-b35d-4d8d-aa03-72100e2889f7" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 636.653031] env[67752]: DEBUG oslo_concurrency.lockutils [None req-29de0487-2264-4436-8ac3-d6e9fd25d33f tempest-ServerTagsTestJSON-923615097 tempest-ServerTagsTestJSON-923615097-project-member] Lock "c38268b5-b35d-4d8d-aa03-72100e2889f7" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 636.741544] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c68aa49d-9279-43b1-b3e4-603b2f38fe16 tempest-AttachVolumeShelveTestJSON-812146230 tempest-AttachVolumeShelveTestJSON-812146230-project-member] Acquiring lock "c6a7f89e-d842-4b34-af36-ca60f12bcddd" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 636.741861] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c68aa49d-9279-43b1-b3e4-603b2f38fe16 tempest-AttachVolumeShelveTestJSON-812146230 tempest-AttachVolumeShelveTestJSON-812146230-project-member] Lock "c6a7f89e-d842-4b34-af36-ca60f12bcddd" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 637.327388] env[67752]: DEBUG nova.network.neutron [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 637.342406] env[67752]: INFO nova.compute.manager [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] Took 0.92 seconds to deallocate network for instance. [ 637.467844] env[67752]: INFO nova.scheduler.client.report [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Deleted allocations for instance 0085fe38-e5ec-44f9-b51b-c54a9f05b60f [ 637.498882] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae10a123-2513-4922-8794-28f1c871ca18 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Lock "0085fe38-e5ec-44f9-b51b-c54a9f05b60f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 64.165s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 637.500957] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "0085fe38-e5ec-44f9-b51b-c54a9f05b60f" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 58.611s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 637.500957] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 0085fe38-e5ec-44f9-b51b-c54a9f05b60f] During sync_power_state the instance has a pending task (spawning). Skip. [ 637.500957] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "0085fe38-e5ec-44f9-b51b-c54a9f05b60f" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 637.534240] env[67752]: DEBUG nova.compute.manager [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 637.592548] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 637.593492] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 637.595637] env[67752]: INFO nova.compute.claims [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 637.958968] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-848be511-6944-4897-943e-6895d2772db7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 637.966922] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd380cdf-73f0-4e8d-971c-105fd39bb3f6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.002597] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9453d665-4d3b-4d54-876c-eaf6c4ac5512 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.010170] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a6ca8e9-8b24-4205-b25e-728644b23adc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.026493] env[67752]: DEBUG nova.compute.provider_tree [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 638.044664] env[67752]: DEBUG nova.scheduler.client.report [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 638.071670] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.478s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 638.072109] env[67752]: DEBUG nova.compute.manager [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 638.117636] env[67752]: DEBUG nova.compute.utils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 638.119912] env[67752]: DEBUG nova.compute.manager [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 638.120169] env[67752]: DEBUG nova.network.neutron [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 638.131325] env[67752]: DEBUG nova.compute.manager [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 638.217097] env[67752]: DEBUG nova.compute.manager [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 638.264560] env[67752]: DEBUG nova.virt.hardware [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 638.264694] env[67752]: DEBUG nova.virt.hardware [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 638.264894] env[67752]: DEBUG nova.virt.hardware [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 638.265340] env[67752]: DEBUG nova.virt.hardware [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 638.266045] env[67752]: DEBUG nova.virt.hardware [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 638.266045] env[67752]: DEBUG nova.virt.hardware [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 638.266045] env[67752]: DEBUG nova.virt.hardware [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 638.266173] env[67752]: DEBUG nova.virt.hardware [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 638.266250] env[67752]: DEBUG nova.virt.hardware [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 638.266436] env[67752]: DEBUG nova.virt.hardware [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 638.266580] env[67752]: DEBUG nova.virt.hardware [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 638.267479] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e5befb74-e5be-4f5e-bde6-c6fc59057f87 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.273371] env[67752]: DEBUG nova.policy [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '2fa41cf9f7054799ba820370515a7105', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3ccbd1f0c92a4b39824dfa66a7d9fb1e', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 638.278744] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fdfef1b7-f26e-46ba-8352-fa88c618a6b1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 638.426246] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2629dfa5-b887-4a2c-bf98-7ac2918140fd tempest-InstanceActionsTestJSON-1707315631 tempest-InstanceActionsTestJSON-1707315631-project-member] Acquiring lock "fa9cdf6d-032f-4069-a78e-fb16b4d2c457" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 638.426570] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2629dfa5-b887-4a2c-bf98-7ac2918140fd tempest-InstanceActionsTestJSON-1707315631 tempest-InstanceActionsTestJSON-1707315631-project-member] Lock "fa9cdf6d-032f-4069-a78e-fb16b4d2c457" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 639.550129] env[67752]: DEBUG nova.network.neutron [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Successfully created port: f92f3c75-049c-42d6-919f-10e0ad2f32a6 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 641.716919] env[67752]: DEBUG nova.network.neutron [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Successfully updated port: f92f3c75-049c-42d6-919f-10e0ad2f32a6 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 641.727443] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Acquiring lock "refresh_cache-eca7c94f-9bd1-4c15-988e-d4e33086ba40" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 641.727597] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Acquired lock "refresh_cache-eca7c94f-9bd1-4c15-988e-d4e33086ba40" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 641.727758] env[67752]: DEBUG nova.network.neutron [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 641.820870] env[67752]: DEBUG nova.network.neutron [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 642.218134] env[67752]: DEBUG nova.network.neutron [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Updating instance_info_cache with network_info: [{"id": "f92f3c75-049c-42d6-919f-10e0ad2f32a6", "address": "fa:16:3e:3b:3f:0c", "network": {"id": "7d7a0d9c-efdc-4005-9dd9-00491f3f4306", "bridge": "br-int", "label": "tempest-ServersTestManualDisk-729089889-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.11", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3ccbd1f0c92a4b39824dfa66a7d9fb1e", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "16f09e8c-5240-4839-80cc-62ec29700bd2", "external-id": "nsx-vlan-transportzone-720", "segmentation_id": 720, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapf92f3c75-04", "ovs_interfaceid": "f92f3c75-049c-42d6-919f-10e0ad2f32a6", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 642.230733] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Releasing lock "refresh_cache-eca7c94f-9bd1-4c15-988e-d4e33086ba40" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 642.231074] env[67752]: DEBUG nova.compute.manager [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Instance network_info: |[{"id": "f92f3c75-049c-42d6-919f-10e0ad2f32a6", "address": "fa:16:3e:3b:3f:0c", "network": {"id": "7d7a0d9c-efdc-4005-9dd9-00491f3f4306", "bridge": "br-int", "label": "tempest-ServersTestManualDisk-729089889-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.11", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3ccbd1f0c92a4b39824dfa66a7d9fb1e", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "16f09e8c-5240-4839-80cc-62ec29700bd2", "external-id": "nsx-vlan-transportzone-720", "segmentation_id": 720, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapf92f3c75-04", "ovs_interfaceid": "f92f3c75-049c-42d6-919f-10e0ad2f32a6", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 642.231495] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:3b:3f:0c', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '16f09e8c-5240-4839-80cc-62ec29700bd2', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'f92f3c75-049c-42d6-919f-10e0ad2f32a6', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 642.245160] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Creating folder: Project (3ccbd1f0c92a4b39824dfa66a7d9fb1e). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 642.245160] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-e4c8498b-fe43-4074-928c-40aa5de90b15 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 642.253322] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Created folder: Project (3ccbd1f0c92a4b39824dfa66a7d9fb1e) in parent group-v639722. [ 642.253527] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Creating folder: Instances. Parent ref: group-v639757. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 642.253763] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-9eb2be31-d6d4-40f6-bc42-899a2d1052ef {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 642.264270] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Created folder: Instances in parent group-v639757. [ 642.264522] env[67752]: DEBUG oslo.service.loopingcall [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 642.264713] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 642.264911] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-1b57f818-0a6a-4c25-afd2-858cbbd9cfa6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 642.291989] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 642.291989] env[67752]: value = "task-3199639" [ 642.291989] env[67752]: _type = "Task" [ 642.291989] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 642.299967] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199639, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 642.801911] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199639, 'name': CreateVM_Task, 'duration_secs': 0.311277} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 642.802657] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 642.803402] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 642.803745] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 642.804030] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 642.804434] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-61c2cf86-8117-4ff4-8d8b-7b5a4fda547d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 642.812270] env[67752]: DEBUG oslo_vmware.api [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Waiting for the task: (returnval){ [ 642.812270] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]521a2203-32e9-6ba2-dffe-56c371c2efa3" [ 642.812270] env[67752]: _type = "Task" [ 642.812270] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 642.826688] env[67752]: DEBUG oslo_vmware.api [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]521a2203-32e9-6ba2-dffe-56c371c2efa3, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 643.157384] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Acquiring lock "580c2aa9-160e-40ec-a1e1-0bbaaf60bca9" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 643.157711] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Lock "580c2aa9-160e-40ec-a1e1-0bbaaf60bca9" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 643.275661] env[67752]: DEBUG oslo_concurrency.lockutils [None req-642b4fb0-db2a-4c92-bc8e-8b8564f22880 tempest-ServerPasswordTestJSON-1994449735 tempest-ServerPasswordTestJSON-1994449735-project-member] Acquiring lock "40aaf487-275a-4e75-ac6a-127371e73b2c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 643.276616] env[67752]: DEBUG oslo_concurrency.lockutils [None req-642b4fb0-db2a-4c92-bc8e-8b8564f22880 tempest-ServerPasswordTestJSON-1994449735 tempest-ServerPasswordTestJSON-1994449735-project-member] Lock "40aaf487-275a-4e75-ac6a-127371e73b2c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 643.306031] env[67752]: DEBUG nova.compute.manager [req-8826611e-d79e-4db7-9b9b-b6269131763c req-62a08cef-780f-410e-86e2-1d6bb2c4f628 service nova] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Received event network-vif-plugged-f92f3c75-049c-42d6-919f-10e0ad2f32a6 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 643.306031] env[67752]: DEBUG oslo_concurrency.lockutils [req-8826611e-d79e-4db7-9b9b-b6269131763c req-62a08cef-780f-410e-86e2-1d6bb2c4f628 service nova] Acquiring lock "eca7c94f-9bd1-4c15-988e-d4e33086ba40-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 643.306031] env[67752]: DEBUG oslo_concurrency.lockutils [req-8826611e-d79e-4db7-9b9b-b6269131763c req-62a08cef-780f-410e-86e2-1d6bb2c4f628 service nova] Lock "eca7c94f-9bd1-4c15-988e-d4e33086ba40-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 643.306031] env[67752]: DEBUG oslo_concurrency.lockutils [req-8826611e-d79e-4db7-9b9b-b6269131763c req-62a08cef-780f-410e-86e2-1d6bb2c4f628 service nova] Lock "eca7c94f-9bd1-4c15-988e-d4e33086ba40-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 643.306177] env[67752]: DEBUG nova.compute.manager [req-8826611e-d79e-4db7-9b9b-b6269131763c req-62a08cef-780f-410e-86e2-1d6bb2c4f628 service nova] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] No waiting events found dispatching network-vif-plugged-f92f3c75-049c-42d6-919f-10e0ad2f32a6 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 643.306177] env[67752]: WARNING nova.compute.manager [req-8826611e-d79e-4db7-9b9b-b6269131763c req-62a08cef-780f-410e-86e2-1d6bb2c4f628 service nova] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Received unexpected event network-vif-plugged-f92f3c75-049c-42d6-919f-10e0ad2f32a6 for instance with vm_state building and task_state spawning. [ 643.326121] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 643.326396] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 643.326638] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 644.880175] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c9fad532-38a2-4412-bcf7-cfbbd2ea1280 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Acquiring lock "d2104921-5ec1-4fb8-8034-52dcdc403a92" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 644.880482] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c9fad532-38a2-4412-bcf7-cfbbd2ea1280 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Lock "d2104921-5ec1-4fb8-8034-52dcdc403a92" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 645.636851] env[67752]: DEBUG nova.compute.manager [req-d48b866d-d00f-4773-a7d2-2c2ada02649b req-33b914d3-c11c-45cc-b010-58b58fbba92e service nova] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Received event network-changed-f92f3c75-049c-42d6-919f-10e0ad2f32a6 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 645.638637] env[67752]: DEBUG nova.compute.manager [req-d48b866d-d00f-4773-a7d2-2c2ada02649b req-33b914d3-c11c-45cc-b010-58b58fbba92e service nova] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Refreshing instance network info cache due to event network-changed-f92f3c75-049c-42d6-919f-10e0ad2f32a6. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 645.638954] env[67752]: DEBUG oslo_concurrency.lockutils [req-d48b866d-d00f-4773-a7d2-2c2ada02649b req-33b914d3-c11c-45cc-b010-58b58fbba92e service nova] Acquiring lock "refresh_cache-eca7c94f-9bd1-4c15-988e-d4e33086ba40" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 645.639119] env[67752]: DEBUG oslo_concurrency.lockutils [req-d48b866d-d00f-4773-a7d2-2c2ada02649b req-33b914d3-c11c-45cc-b010-58b58fbba92e service nova] Acquired lock "refresh_cache-eca7c94f-9bd1-4c15-988e-d4e33086ba40" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 645.639286] env[67752]: DEBUG nova.network.neutron [req-d48b866d-d00f-4773-a7d2-2c2ada02649b req-33b914d3-c11c-45cc-b010-58b58fbba92e service nova] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Refreshing network info cache for port f92f3c75-049c-42d6-919f-10e0ad2f32a6 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 646.485946] env[67752]: DEBUG nova.network.neutron [req-d48b866d-d00f-4773-a7d2-2c2ada02649b req-33b914d3-c11c-45cc-b010-58b58fbba92e service nova] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Updated VIF entry in instance network info cache for port f92f3c75-049c-42d6-919f-10e0ad2f32a6. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 646.486344] env[67752]: DEBUG nova.network.neutron [req-d48b866d-d00f-4773-a7d2-2c2ada02649b req-33b914d3-c11c-45cc-b010-58b58fbba92e service nova] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Updating instance_info_cache with network_info: [{"id": "f92f3c75-049c-42d6-919f-10e0ad2f32a6", "address": "fa:16:3e:3b:3f:0c", "network": {"id": "7d7a0d9c-efdc-4005-9dd9-00491f3f4306", "bridge": "br-int", "label": "tempest-ServersTestManualDisk-729089889-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.11", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3ccbd1f0c92a4b39824dfa66a7d9fb1e", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "16f09e8c-5240-4839-80cc-62ec29700bd2", "external-id": "nsx-vlan-transportzone-720", "segmentation_id": 720, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapf92f3c75-04", "ovs_interfaceid": "f92f3c75-049c-42d6-919f-10e0ad2f32a6", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 646.503532] env[67752]: DEBUG oslo_concurrency.lockutils [req-d48b866d-d00f-4773-a7d2-2c2ada02649b req-33b914d3-c11c-45cc-b010-58b58fbba92e service nova] Releasing lock "refresh_cache-eca7c94f-9bd1-4c15-988e-d4e33086ba40" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 646.971790] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fa6ab7e9-8f79-414e-8999-6de8d04f8b75 tempest-AttachInterfacesUnderV243Test-278406362 tempest-AttachInterfacesUnderV243Test-278406362-project-member] Acquiring lock "16f0124d-146a-442a-8738-65f562b928e8" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 646.971790] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fa6ab7e9-8f79-414e-8999-6de8d04f8b75 tempest-AttachInterfacesUnderV243Test-278406362 tempest-AttachInterfacesUnderV243Test-278406362-project-member] Lock "16f0124d-146a-442a-8738-65f562b928e8" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 648.445984] env[67752]: WARNING oslo_vmware.rw_handles [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 648.445984] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 648.445984] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 648.445984] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 648.445984] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 648.445984] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 648.445984] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 648.445984] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 648.445984] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 648.445984] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 648.445984] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 648.445984] env[67752]: ERROR oslo_vmware.rw_handles [ 648.445984] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/12554a4c-c719-440f-b42e-de8c370be4d6/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore1 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 648.446796] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 648.446796] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Copying Virtual Disk [datastore1] vmware_temp/12554a4c-c719-440f-b42e-de8c370be4d6/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore1] vmware_temp/12554a4c-c719-440f-b42e-de8c370be4d6/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 648.446796] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-7422cc59-c1e1-4cc1-897b-e122ed07269b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.459802] env[67752]: DEBUG oslo_vmware.api [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Waiting for the task: (returnval){ [ 648.459802] env[67752]: value = "task-3199640" [ 648.459802] env[67752]: _type = "Task" [ 648.459802] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 648.470284] env[67752]: DEBUG oslo_vmware.api [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Task: {'id': task-3199640, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 648.971903] env[67752]: DEBUG oslo_vmware.exceptions [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 648.972344] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Releasing lock "[datastore1] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 648.973257] env[67752]: ERROR nova.compute.manager [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 648.973257] env[67752]: Faults: ['InvalidArgument'] [ 648.973257] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Traceback (most recent call last): [ 648.973257] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 648.973257] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] yield resources [ 648.973257] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 648.973257] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] self.driver.spawn(context, instance, image_meta, [ 648.973257] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 648.973257] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 648.973257] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 648.973257] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] self._fetch_image_if_missing(context, vi) [ 648.973257] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 648.973696] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] image_cache(vi, tmp_image_ds_loc) [ 648.973696] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 648.973696] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] vm_util.copy_virtual_disk( [ 648.973696] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 648.973696] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] session._wait_for_task(vmdk_copy_task) [ 648.973696] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 648.973696] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] return self.wait_for_task(task_ref) [ 648.973696] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 648.973696] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] return evt.wait() [ 648.973696] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 648.973696] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] result = hub.switch() [ 648.973696] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 648.973696] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] return self.greenlet.switch() [ 648.974138] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 648.974138] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] self.f(*self.args, **self.kw) [ 648.974138] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 648.974138] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] raise exceptions.translate_fault(task_info.error) [ 648.974138] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 648.974138] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Faults: ['InvalidArgument'] [ 648.974138] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] [ 648.974138] env[67752]: INFO nova.compute.manager [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Terminating instance [ 648.976697] env[67752]: DEBUG nova.compute.manager [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 648.976901] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 648.977705] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f051087-f9ae-46be-9b5b-f04a425de083 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 648.985765] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 648.985880] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-21d05384-ce3d-42e0-8c44-837f0add8a18 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 649.324255] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 649.324499] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Deleting contents of the VM from datastore datastore1 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 649.324687] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Deleting the datastore file [datastore1] fd0cd82b-9339-4415-aa30-af19f46fd3e0 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 649.324957] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-a9270194-703e-4e7e-84cb-c2b1fb4c4f00 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 649.337154] env[67752]: DEBUG oslo_vmware.api [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Waiting for the task: (returnval){ [ 649.337154] env[67752]: value = "task-3199642" [ 649.337154] env[67752]: _type = "Task" [ 649.337154] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 649.348146] env[67752]: DEBUG oslo_vmware.api [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Task: {'id': task-3199642, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 649.847396] env[67752]: DEBUG oslo_vmware.api [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Task: {'id': task-3199642, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.074371} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 649.847666] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 649.847822] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Deleted contents of the VM from datastore datastore1 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 649.848010] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 649.848487] env[67752]: INFO nova.compute.manager [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Took 0.87 seconds to destroy the instance on the hypervisor. [ 649.850215] env[67752]: DEBUG nova.compute.claims [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 649.850792] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 649.850792] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 650.295248] env[67752]: DEBUG oslo_concurrency.lockutils [None req-66e56245-c32a-4a6d-8492-e81780b14c1e tempest-ListServerFiltersTestJSON-1646144619 tempest-ListServerFiltersTestJSON-1646144619-project-member] Acquiring lock "8503063e-2888-44c6-bc2a-e9c82be791e2" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 650.295512] env[67752]: DEBUG oslo_concurrency.lockutils [None req-66e56245-c32a-4a6d-8492-e81780b14c1e tempest-ListServerFiltersTestJSON-1646144619 tempest-ListServerFiltersTestJSON-1646144619-project-member] Lock "8503063e-2888-44c6-bc2a-e9c82be791e2" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 650.466089] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b68f819f-85af-4e43-9070-531f6639c421 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.476184] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0cda06d5-7229-4039-9191-1ab5b6c6b493 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.513180] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d792f0a8-5b39-4ec8-8d73-a3229b08bff2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.520913] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2e694b1c-378b-4f6a-a746-f4d58686d517 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 650.540724] env[67752]: DEBUG nova.compute.provider_tree [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 650.555488] env[67752]: DEBUG nova.scheduler.client.report [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 650.583172] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.732s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 650.583172] env[67752]: ERROR nova.compute.manager [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 650.583172] env[67752]: Faults: ['InvalidArgument'] [ 650.583172] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Traceback (most recent call last): [ 650.583172] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 650.583172] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] self.driver.spawn(context, instance, image_meta, [ 650.583172] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 650.583172] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] self._vmops.spawn(context, instance, image_meta, injected_files, [ 650.583172] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 650.583172] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] self._fetch_image_if_missing(context, vi) [ 650.584053] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 650.584053] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] image_cache(vi, tmp_image_ds_loc) [ 650.584053] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 650.584053] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] vm_util.copy_virtual_disk( [ 650.584053] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 650.584053] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] session._wait_for_task(vmdk_copy_task) [ 650.584053] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 650.584053] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] return self.wait_for_task(task_ref) [ 650.584053] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 650.584053] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] return evt.wait() [ 650.584053] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 650.584053] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] result = hub.switch() [ 650.584053] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 650.584550] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] return self.greenlet.switch() [ 650.584550] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 650.584550] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] self.f(*self.args, **self.kw) [ 650.584550] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 650.584550] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] raise exceptions.translate_fault(task_info.error) [ 650.584550] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 650.584550] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Faults: ['InvalidArgument'] [ 650.584550] env[67752]: ERROR nova.compute.manager [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] [ 650.584550] env[67752]: DEBUG nova.compute.utils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 650.588873] env[67752]: DEBUG nova.compute.manager [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Build of instance fd0cd82b-9339-4415-aa30-af19f46fd3e0 was re-scheduled: A specified parameter was not correct: fileType [ 650.588873] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 650.588873] env[67752]: DEBUG nova.compute.manager [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 650.588873] env[67752]: DEBUG nova.compute.manager [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 650.588873] env[67752]: DEBUG nova.compute.manager [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 650.589308] env[67752]: DEBUG nova.network.neutron [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 651.553683] env[67752]: DEBUG nova.network.neutron [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 651.568124] env[67752]: INFO nova.compute.manager [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] [instance: fd0cd82b-9339-4415-aa30-af19f46fd3e0] Took 0.98 seconds to deallocate network for instance. [ 651.731380] env[67752]: INFO nova.scheduler.client.report [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Deleted allocations for instance fd0cd82b-9339-4415-aa30-af19f46fd3e0 [ 651.765504] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0b113a98-69d4-4c19-9e5b-c1e942b6ae7b tempest-ImagesOneServerNegativeTestJSON-795755565 tempest-ImagesOneServerNegativeTestJSON-795755565-project-member] Lock "fd0cd82b-9339-4415-aa30-af19f46fd3e0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 59.695s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 651.783156] env[67752]: DEBUG nova.compute.manager [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 651.862694] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 651.863617] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 651.866566] env[67752]: INFO nova.compute.claims [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 652.384882] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d5d7b30b-6b41-488b-b26f-fee8498fa111 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.395981] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf565954-b7ca-4f58-9102-78d7eb656388 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.431963] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f045e4a3-9eee-4f95-b186-15c95d1ef409 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.439564] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e197ef50-0091-4351-aea8-9bd068b21cfb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.457753] env[67752]: DEBUG nova.compute.provider_tree [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 652.472126] env[67752]: DEBUG nova.scheduler.client.report [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 652.499163] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.635s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 652.499498] env[67752]: DEBUG nova.compute.manager [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 652.552206] env[67752]: DEBUG nova.compute.utils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 652.553363] env[67752]: DEBUG nova.compute.manager [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 652.553514] env[67752]: DEBUG nova.network.neutron [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 652.565410] env[67752]: DEBUG nova.compute.manager [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 652.588186] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ec3ed760-d07f-4b6f-b123-835949de2da9 tempest-ListServerFiltersTestJSON-1646144619 tempest-ListServerFiltersTestJSON-1646144619-project-member] Acquiring lock "631002d2-c268-4a6a-9945-c0898b886b93" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 652.588431] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ec3ed760-d07f-4b6f-b123-835949de2da9 tempest-ListServerFiltersTestJSON-1646144619 tempest-ListServerFiltersTestJSON-1646144619-project-member] Lock "631002d2-c268-4a6a-9945-c0898b886b93" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 652.619201] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 652.651550] env[67752]: DEBUG nova.compute.manager [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 652.666424] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 652.666793] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 652.666971] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 652.690259] env[67752]: DEBUG nova.virt.hardware [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 652.690696] env[67752]: DEBUG nova.virt.hardware [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 652.690696] env[67752]: DEBUG nova.virt.hardware [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 652.690909] env[67752]: DEBUG nova.virt.hardware [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 652.691122] env[67752]: DEBUG nova.virt.hardware [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 652.691338] env[67752]: DEBUG nova.virt.hardware [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 652.691648] env[67752]: DEBUG nova.virt.hardware [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 652.691648] env[67752]: DEBUG nova.virt.hardware [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 652.691732] env[67752]: DEBUG nova.virt.hardware [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 652.691891] env[67752]: DEBUG nova.virt.hardware [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 652.692094] env[67752]: DEBUG nova.virt.hardware [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 652.692995] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-03d37350-31b6-44c5-9211-c61f4228e02a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.698519] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 652.698688] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 652.699394] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 652.699692] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 652.699782] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 652.699951] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 652.700075] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 652.700227] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 652.700440] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 652.700624] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 652.700689] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 652.701715] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 652.701874] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 652.702062] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 652.711575] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-70a08115-47fd-4f26-84d2-31d98f2168b2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.727947] env[67752]: DEBUG nova.policy [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '6c1e3aad166a4d5c8de9c632b2ab2139', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3a36f872b5c044a5a0dcbce1e454fcd9', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 652.732560] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 652.732560] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 652.732721] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 652.732943] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 652.734022] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad4eb52b-717b-4f5e-b74c-dafa6de3e70c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.745673] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-13d2bf67-e337-484b-8e7d-ed9e78f79427 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.762653] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b7ca1eb7-b7eb-43b2-a02e-fde11b8ff40e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.773561] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ad9dd20-c258-4ecd-83f1-7607ef03ff37 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 652.809228] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180997MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 652.809404] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 652.809616] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 652.893988] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 652.894183] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c563b7ed-4fb3-4fb0-8350-789fbe4d53a2 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 652.894318] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fc4da5fa-1921-44e3-9100-042391d4ccb9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 652.894446] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b5eee51b-73b8-4e17-98a4-e108a5b21892 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 652.894623] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 39479042-b891-4f25-9992-47312c6b6b43 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 652.895521] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 652.895521] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 652.895625] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b0cc1acd-0e33-4547-a43b-549b3de81728 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 652.895727] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance eca7c94f-9bd1-4c15-988e-d4e33086ba40 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 652.896176] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ff4895f8-0374-4b34-a5e8-d3ba24f840a5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 652.917570] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ca18af6b-6b80-44f3-bd8b-f61e2596acd7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 652.949077] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance cbe3fe72-5a91-4c81-ac62-be6592aa806a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 652.963537] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 02942b29-2343-441e-9e2b-aca2af57c849 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 652.978880] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ebd6c719-32ae-410c-a444-db7ad3f8a9c5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 652.996380] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f528dc15-1a86-4af5-ac45-536fe3dfe101 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 653.011395] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 4c0873a7-b8e5-4d3e-bdc0-3d236e3b9df6 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 653.028119] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e3c2e1cf-cae5-474f-9f61-96d79815d846 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 653.042736] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f22bcbae-0dcb-4666-b5e7-00c560e016c3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 653.057566] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c38268b5-b35d-4d8d-aa03-72100e2889f7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 653.094383] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c6a7f89e-d842-4b34-af36-ca60f12bcddd has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 653.109490] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fa9cdf6d-032f-4069-a78e-fb16b4d2c457 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 653.132760] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 653.149147] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 40aaf487-275a-4e75-ac6a-127371e73b2c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 653.164833] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d2104921-5ec1-4fb8-8034-52dcdc403a92 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 653.181952] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 16f0124d-146a-442a-8738-65f562b928e8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 653.200476] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 8503063e-2888-44c6-bc2a-e9c82be791e2 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 653.213155] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 631002d2-c268-4a6a-9945-c0898b886b93 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 653.213376] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 653.213610] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 653.264400] env[67752]: DEBUG oslo_concurrency.lockutils [None req-162ff140-7e34-4099-9735-3cd37bbd4868 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Acquiring lock "67812e2e-8827-4368-9790-a909c5ebb56b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 653.264532] env[67752]: DEBUG oslo_concurrency.lockutils [None req-162ff140-7e34-4099-9735-3cd37bbd4868 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Lock "67812e2e-8827-4368-9790-a909c5ebb56b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 653.636471] env[67752]: DEBUG nova.network.neutron [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Successfully created port: 42acd164-c40d-457a-9918-50595ae5a65b {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 653.688998] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-738b8439-228f-4fbc-9dad-411e3a81ac15 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.697977] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eccda483-1c89-4582-82db-35fd3ebb028b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.732163] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8555bb6-107c-4d9e-a9d6-5bc303db9cef {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.744192] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-175d8890-ad6d-4b26-9c41-1a98c3be7528 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 653.762221] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 653.784868] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 653.803666] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 653.804012] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.994s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 654.094792] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5b072838-12e6-4e37-966a-8e553b120493 tempest-ServerActionsTestJSON-376351690 tempest-ServerActionsTestJSON-376351690-project-member] Acquiring lock "5ed08031-0ef8-4766-951a-37b315c55dc7" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 654.094938] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5b072838-12e6-4e37-966a-8e553b120493 tempest-ServerActionsTestJSON-376351690 tempest-ServerActionsTestJSON-376351690-project-member] Lock "5ed08031-0ef8-4766-951a-37b315c55dc7" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 654.736876] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 654.737129] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 654.738029] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 654.738029] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 654.738029] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 654.738029] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 654.849456] env[67752]: DEBUG nova.network.neutron [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Successfully updated port: 42acd164-c40d-457a-9918-50595ae5a65b {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 654.863480] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquiring lock "refresh_cache-ff4895f8-0374-4b34-a5e8-d3ba24f840a5" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 654.865279] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquired lock "refresh_cache-ff4895f8-0374-4b34-a5e8-d3ba24f840a5" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 654.865279] env[67752]: DEBUG nova.network.neutron [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 654.944361] env[67752]: DEBUG nova.network.neutron [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 655.046245] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5ebe9f01-ba51-454e-a9fb-85af4a904115 tempest-ListServerFiltersTestJSON-1646144619 tempest-ListServerFiltersTestJSON-1646144619-project-member] Acquiring lock "436d243a-d6d9-4958-94a8-7b09f5d9fdf3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 655.046396] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5ebe9f01-ba51-454e-a9fb-85af4a904115 tempest-ListServerFiltersTestJSON-1646144619 tempest-ListServerFiltersTestJSON-1646144619-project-member] Lock "436d243a-d6d9-4958-94a8-7b09f5d9fdf3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 655.510518] env[67752]: DEBUG nova.network.neutron [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Updating instance_info_cache with network_info: [{"id": "42acd164-c40d-457a-9918-50595ae5a65b", "address": "fa:16:3e:03:cd:be", "network": {"id": "8754f810-4855-45bb-9a57-6c1511f0722f", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.18", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "c43e0a2618b34312898e0e4bb014b17c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "983826cf-6390-4ec6-bf97-30a1060947fc", "external-id": "nsx-vlan-transportzone-367", "segmentation_id": 367, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap42acd164-c4", "ovs_interfaceid": "42acd164-c40d-457a-9918-50595ae5a65b", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 655.525697] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Releasing lock "refresh_cache-ff4895f8-0374-4b34-a5e8-d3ba24f840a5" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 655.526025] env[67752]: DEBUG nova.compute.manager [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Instance network_info: |[{"id": "42acd164-c40d-457a-9918-50595ae5a65b", "address": "fa:16:3e:03:cd:be", "network": {"id": "8754f810-4855-45bb-9a57-6c1511f0722f", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.18", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "c43e0a2618b34312898e0e4bb014b17c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "983826cf-6390-4ec6-bf97-30a1060947fc", "external-id": "nsx-vlan-transportzone-367", "segmentation_id": 367, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap42acd164-c4", "ovs_interfaceid": "42acd164-c40d-457a-9918-50595ae5a65b", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 655.526478] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:03:cd:be', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '983826cf-6390-4ec6-bf97-30a1060947fc', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '42acd164-c40d-457a-9918-50595ae5a65b', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 655.534037] env[67752]: DEBUG oslo.service.loopingcall [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 655.534505] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 655.534735] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-cacd392d-10d0-4369-82ac-5e2b74be8f1f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 655.555056] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 655.555056] env[67752]: value = "task-3199643" [ 655.555056] env[67752]: _type = "Task" [ 655.555056] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 655.563748] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199643, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 656.067362] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199643, 'name': CreateVM_Task, 'duration_secs': 0.311261} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 656.071017] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 656.071017] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 656.071017] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 656.071017] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 656.071017] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b41c9947-58a0-4fc4-9de3-a1c54300bdd1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 656.073736] env[67752]: DEBUG oslo_vmware.api [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Waiting for the task: (returnval){ [ 656.073736] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52b401ab-a138-9f61-69d8-b1ac289be1b9" [ 656.073736] env[67752]: _type = "Task" [ 656.073736] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 656.081951] env[67752]: DEBUG oslo_vmware.api [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52b401ab-a138-9f61-69d8-b1ac289be1b9, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 656.261139] env[67752]: DEBUG nova.compute.manager [req-1e8aeb9f-d6a6-4d15-aa04-c2e12955bed9 req-bbb7b2e9-3ff1-4117-b3fc-f5db857d684e service nova] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Received event network-vif-plugged-42acd164-c40d-457a-9918-50595ae5a65b {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 656.261401] env[67752]: DEBUG oslo_concurrency.lockutils [req-1e8aeb9f-d6a6-4d15-aa04-c2e12955bed9 req-bbb7b2e9-3ff1-4117-b3fc-f5db857d684e service nova] Acquiring lock "ff4895f8-0374-4b34-a5e8-d3ba24f840a5-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 656.262463] env[67752]: DEBUG oslo_concurrency.lockutils [req-1e8aeb9f-d6a6-4d15-aa04-c2e12955bed9 req-bbb7b2e9-3ff1-4117-b3fc-f5db857d684e service nova] Lock "ff4895f8-0374-4b34-a5e8-d3ba24f840a5-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 656.262673] env[67752]: DEBUG oslo_concurrency.lockutils [req-1e8aeb9f-d6a6-4d15-aa04-c2e12955bed9 req-bbb7b2e9-3ff1-4117-b3fc-f5db857d684e service nova] Lock "ff4895f8-0374-4b34-a5e8-d3ba24f840a5-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 656.262856] env[67752]: DEBUG nova.compute.manager [req-1e8aeb9f-d6a6-4d15-aa04-c2e12955bed9 req-bbb7b2e9-3ff1-4117-b3fc-f5db857d684e service nova] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] No waiting events found dispatching network-vif-plugged-42acd164-c40d-457a-9918-50595ae5a65b {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 656.263041] env[67752]: WARNING nova.compute.manager [req-1e8aeb9f-d6a6-4d15-aa04-c2e12955bed9 req-bbb7b2e9-3ff1-4117-b3fc-f5db857d684e service nova] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Received unexpected event network-vif-plugged-42acd164-c40d-457a-9918-50595ae5a65b for instance with vm_state building and task_state spawning. [ 656.263213] env[67752]: DEBUG nova.compute.manager [req-1e8aeb9f-d6a6-4d15-aa04-c2e12955bed9 req-bbb7b2e9-3ff1-4117-b3fc-f5db857d684e service nova] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Received event network-changed-42acd164-c40d-457a-9918-50595ae5a65b {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 656.263371] env[67752]: DEBUG nova.compute.manager [req-1e8aeb9f-d6a6-4d15-aa04-c2e12955bed9 req-bbb7b2e9-3ff1-4117-b3fc-f5db857d684e service nova] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Refreshing instance network info cache due to event network-changed-42acd164-c40d-457a-9918-50595ae5a65b. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 656.263555] env[67752]: DEBUG oslo_concurrency.lockutils [req-1e8aeb9f-d6a6-4d15-aa04-c2e12955bed9 req-bbb7b2e9-3ff1-4117-b3fc-f5db857d684e service nova] Acquiring lock "refresh_cache-ff4895f8-0374-4b34-a5e8-d3ba24f840a5" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 656.263691] env[67752]: DEBUG oslo_concurrency.lockutils [req-1e8aeb9f-d6a6-4d15-aa04-c2e12955bed9 req-bbb7b2e9-3ff1-4117-b3fc-f5db857d684e service nova] Acquired lock "refresh_cache-ff4895f8-0374-4b34-a5e8-d3ba24f840a5" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 656.263845] env[67752]: DEBUG nova.network.neutron [req-1e8aeb9f-d6a6-4d15-aa04-c2e12955bed9 req-bbb7b2e9-3ff1-4117-b3fc-f5db857d684e service nova] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Refreshing network info cache for port 42acd164-c40d-457a-9918-50595ae5a65b {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 656.583937] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 656.584251] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 656.584514] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 656.770925] env[67752]: DEBUG nova.network.neutron [req-1e8aeb9f-d6a6-4d15-aa04-c2e12955bed9 req-bbb7b2e9-3ff1-4117-b3fc-f5db857d684e service nova] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Updated VIF entry in instance network info cache for port 42acd164-c40d-457a-9918-50595ae5a65b. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 656.771357] env[67752]: DEBUG nova.network.neutron [req-1e8aeb9f-d6a6-4d15-aa04-c2e12955bed9 req-bbb7b2e9-3ff1-4117-b3fc-f5db857d684e service nova] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Updating instance_info_cache with network_info: [{"id": "42acd164-c40d-457a-9918-50595ae5a65b", "address": "fa:16:3e:03:cd:be", "network": {"id": "8754f810-4855-45bb-9a57-6c1511f0722f", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.18", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "c43e0a2618b34312898e0e4bb014b17c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "983826cf-6390-4ec6-bf97-30a1060947fc", "external-id": "nsx-vlan-transportzone-367", "segmentation_id": 367, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap42acd164-c4", "ovs_interfaceid": "42acd164-c40d-457a-9918-50595ae5a65b", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 656.783690] env[67752]: DEBUG oslo_concurrency.lockutils [req-1e8aeb9f-d6a6-4d15-aa04-c2e12955bed9 req-bbb7b2e9-3ff1-4117-b3fc-f5db857d684e service nova] Releasing lock "refresh_cache-ff4895f8-0374-4b34-a5e8-d3ba24f840a5" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 660.464823] env[67752]: DEBUG oslo_concurrency.lockutils [None req-49fb07d2-4df8-4b3a-99c6-543da3a9498a tempest-ServersWithSpecificFlavorTestJSON-1001786271 tempest-ServersWithSpecificFlavorTestJSON-1001786271-project-member] Acquiring lock "f8b03251-6f23-4533-a917-f3fdac87888b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 660.465168] env[67752]: DEBUG oslo_concurrency.lockutils [None req-49fb07d2-4df8-4b3a-99c6-543da3a9498a tempest-ServersWithSpecificFlavorTestJSON-1001786271 tempest-ServersWithSpecificFlavorTestJSON-1001786271-project-member] Lock "f8b03251-6f23-4533-a917-f3fdac87888b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 664.189076] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c63e4090-88c0-446e-b77e-592b55cc7924 tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] Acquiring lock "58d3b02f-3aac-4beb-a111-44f50c7f1bc8" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 664.189666] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c63e4090-88c0-446e-b77e-592b55cc7924 tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] Lock "58d3b02f-3aac-4beb-a111-44f50c7f1bc8" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 671.597772] env[67752]: DEBUG oslo_concurrency.lockutils [None req-060466ab-7897-4299-ab80-97f57b46db01 tempest-ServersTestFqdnHostnames-644753716 tempest-ServersTestFqdnHostnames-644753716-project-member] Acquiring lock "53eb8b0c-aa50-4d7f-8027-2725c815f134" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 671.598087] env[67752]: DEBUG oslo_concurrency.lockutils [None req-060466ab-7897-4299-ab80-97f57b46db01 tempest-ServersTestFqdnHostnames-644753716 tempest-ServersTestFqdnHostnames-644753716-project-member] Lock "53eb8b0c-aa50-4d7f-8027-2725c815f134" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 683.406097] env[67752]: WARNING oslo_vmware.rw_handles [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 683.406097] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 683.406097] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 683.406097] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 683.406097] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 683.406097] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 683.406097] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 683.406097] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 683.406097] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 683.406097] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 683.406097] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 683.406097] env[67752]: ERROR oslo_vmware.rw_handles [ 683.406097] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/9b796c90-1b4f-4a7f-8778-a0c0267d2797/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 683.407845] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 683.408532] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Copying Virtual Disk [datastore2] vmware_temp/9b796c90-1b4f-4a7f-8778-a0c0267d2797/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/9b796c90-1b4f-4a7f-8778-a0c0267d2797/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 683.408532] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-bd9d507d-8a82-4a8d-bd21-1afb20071cf8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 683.418547] env[67752]: DEBUG oslo_vmware.api [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Waiting for the task: (returnval){ [ 683.418547] env[67752]: value = "task-3199644" [ 683.418547] env[67752]: _type = "Task" [ 683.418547] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 683.426382] env[67752]: DEBUG oslo_vmware.api [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Task: {'id': task-3199644, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 683.929563] env[67752]: DEBUG oslo_vmware.exceptions [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 683.929841] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 683.930416] env[67752]: ERROR nova.compute.manager [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 683.930416] env[67752]: Faults: ['InvalidArgument'] [ 683.930416] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Traceback (most recent call last): [ 683.930416] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 683.930416] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] yield resources [ 683.930416] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 683.930416] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] self.driver.spawn(context, instance, image_meta, [ 683.930416] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 683.930416] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 683.930416] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 683.930416] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] self._fetch_image_if_missing(context, vi) [ 683.930416] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 683.930714] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] image_cache(vi, tmp_image_ds_loc) [ 683.930714] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 683.930714] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] vm_util.copy_virtual_disk( [ 683.930714] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 683.930714] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] session._wait_for_task(vmdk_copy_task) [ 683.930714] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 683.930714] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] return self.wait_for_task(task_ref) [ 683.930714] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 683.930714] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] return evt.wait() [ 683.930714] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 683.930714] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] result = hub.switch() [ 683.930714] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 683.930714] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] return self.greenlet.switch() [ 683.930979] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 683.930979] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] self.f(*self.args, **self.kw) [ 683.930979] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 683.930979] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] raise exceptions.translate_fault(task_info.error) [ 683.930979] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 683.930979] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Faults: ['InvalidArgument'] [ 683.930979] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] [ 683.930979] env[67752]: INFO nova.compute.manager [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Terminating instance [ 683.932347] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 683.932547] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 683.932777] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-c1372f4c-4f5f-42c6-904e-2e302122a222 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 683.934784] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Acquiring lock "refresh_cache-c563b7ed-4fb3-4fb0-8350-789fbe4d53a2" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 683.934948] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Acquired lock "refresh_cache-c563b7ed-4fb3-4fb0-8350-789fbe4d53a2" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 683.935128] env[67752]: DEBUG nova.network.neutron [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 683.943366] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 683.943617] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 683.945342] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b9776d99-9830-4d45-98f5-62aa783319d0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 683.954385] env[67752]: DEBUG oslo_vmware.api [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Waiting for the task: (returnval){ [ 683.954385] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52f1f418-165b-1c5c-02ba-dbc70fdfe6d9" [ 683.954385] env[67752]: _type = "Task" [ 683.954385] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 683.962405] env[67752]: DEBUG oslo_vmware.api [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52f1f418-165b-1c5c-02ba-dbc70fdfe6d9, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 683.972793] env[67752]: DEBUG nova.network.neutron [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 684.136829] env[67752]: DEBUG nova.network.neutron [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 684.145931] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Releasing lock "refresh_cache-c563b7ed-4fb3-4fb0-8350-789fbe4d53a2" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 684.146350] env[67752]: DEBUG nova.compute.manager [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 684.146538] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 684.147616] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d41c2032-303d-4597-a4ab-daed53ce57f9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 684.155724] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 684.155940] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-e7004f57-4452-4d7c-b2e3-94494ae23222 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 684.184281] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 684.184604] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 684.185805] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Deleting the datastore file [datastore2] c563b7ed-4fb3-4fb0-8350-789fbe4d53a2 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 684.185805] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-e6b3691c-b87c-4c31-8060-755b647734c2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 684.192235] env[67752]: DEBUG oslo_vmware.api [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Waiting for the task: (returnval){ [ 684.192235] env[67752]: value = "task-3199646" [ 684.192235] env[67752]: _type = "Task" [ 684.192235] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 684.199835] env[67752]: DEBUG oslo_vmware.api [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Task: {'id': task-3199646, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 684.466244] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 684.466520] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Creating directory with path [datastore2] vmware_temp/457c77fd-4e5d-4e92-9ba7-602bedf0e43f/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 684.466727] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-fb8789ca-2bca-4ce0-86ad-31b1481553fa {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 684.478296] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Created directory with path [datastore2] vmware_temp/457c77fd-4e5d-4e92-9ba7-602bedf0e43f/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 684.478489] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Fetch image to [datastore2] vmware_temp/457c77fd-4e5d-4e92-9ba7-602bedf0e43f/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 684.478656] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/457c77fd-4e5d-4e92-9ba7-602bedf0e43f/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 684.479404] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bda4a199-d928-42c4-b076-68f07eb74acb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 684.486237] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e5f2f977-1235-48f9-9de7-03837a094766 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 684.495256] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-02026822-ad41-4346-b0d9-66b22db2e0c5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 684.525313] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08e0ca05-d16f-4e3b-a6a5-cf8c46e362d4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 684.531053] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-3f2dc52d-b3e4-4802-914b-6d11710d7838 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 684.550609] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 684.601096] env[67752]: DEBUG oslo_vmware.rw_handles [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/457c77fd-4e5d-4e92-9ba7-602bedf0e43f/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 684.660741] env[67752]: DEBUG oslo_vmware.rw_handles [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 684.660903] env[67752]: DEBUG oslo_vmware.rw_handles [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Closing write handle for https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/457c77fd-4e5d-4e92-9ba7-602bedf0e43f/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 684.702067] env[67752]: DEBUG oslo_vmware.api [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Task: {'id': task-3199646, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.049097} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 684.702263] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 684.702456] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 684.702635] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 684.702810] env[67752]: INFO nova.compute.manager [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Took 0.56 seconds to destroy the instance on the hypervisor. [ 684.703060] env[67752]: DEBUG oslo.service.loopingcall [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 684.703278] env[67752]: DEBUG nova.compute.manager [-] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Skipping network deallocation for instance since networking was not requested. {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2276}} [ 684.705429] env[67752]: DEBUG nova.compute.claims [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 684.705603] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 684.705817] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 685.139787] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2372d65e-37f2-4f61-8c3f-3f1add657427 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 685.147800] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a4c463c1-65b0-4249-9c6b-188672f3e74b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 685.177912] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-66f846e8-08fe-464b-a445-383dcb10e4b5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 685.184866] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2fb8cdda-82cb-48c1-ad10-3337de236995 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 685.198903] env[67752]: DEBUG nova.compute.provider_tree [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 685.208594] env[67752]: DEBUG nova.scheduler.client.report [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 685.226107] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.520s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 685.226651] env[67752]: ERROR nova.compute.manager [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 685.226651] env[67752]: Faults: ['InvalidArgument'] [ 685.226651] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Traceback (most recent call last): [ 685.226651] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 685.226651] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] self.driver.spawn(context, instance, image_meta, [ 685.226651] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 685.226651] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] self._vmops.spawn(context, instance, image_meta, injected_files, [ 685.226651] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 685.226651] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] self._fetch_image_if_missing(context, vi) [ 685.226651] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 685.226651] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] image_cache(vi, tmp_image_ds_loc) [ 685.226651] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 685.226933] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] vm_util.copy_virtual_disk( [ 685.226933] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 685.226933] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] session._wait_for_task(vmdk_copy_task) [ 685.226933] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 685.226933] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] return self.wait_for_task(task_ref) [ 685.226933] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 685.226933] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] return evt.wait() [ 685.226933] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 685.226933] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] result = hub.switch() [ 685.226933] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 685.226933] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] return self.greenlet.switch() [ 685.226933] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 685.226933] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] self.f(*self.args, **self.kw) [ 685.227203] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 685.227203] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] raise exceptions.translate_fault(task_info.error) [ 685.227203] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 685.227203] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Faults: ['InvalidArgument'] [ 685.227203] env[67752]: ERROR nova.compute.manager [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] [ 685.227367] env[67752]: DEBUG nova.compute.utils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 685.228778] env[67752]: DEBUG nova.compute.manager [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Build of instance c563b7ed-4fb3-4fb0-8350-789fbe4d53a2 was re-scheduled: A specified parameter was not correct: fileType [ 685.228778] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 685.229173] env[67752]: DEBUG nova.compute.manager [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 685.229401] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Acquiring lock "refresh_cache-c563b7ed-4fb3-4fb0-8350-789fbe4d53a2" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 685.229549] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Acquired lock "refresh_cache-c563b7ed-4fb3-4fb0-8350-789fbe4d53a2" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 685.229709] env[67752]: DEBUG nova.network.neutron [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 685.270589] env[67752]: DEBUG nova.network.neutron [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 685.388185] env[67752]: DEBUG nova.network.neutron [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 685.398990] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Releasing lock "refresh_cache-c563b7ed-4fb3-4fb0-8350-789fbe4d53a2" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 685.399225] env[67752]: DEBUG nova.compute.manager [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 685.399409] env[67752]: DEBUG nova.compute.manager [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] [instance: c563b7ed-4fb3-4fb0-8350-789fbe4d53a2] Skipping network deallocation for instance since networking was not requested. {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2276}} [ 685.490819] env[67752]: INFO nova.scheduler.client.report [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Deleted allocations for instance c563b7ed-4fb3-4fb0-8350-789fbe4d53a2 [ 685.509383] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7eeea111-1030-45d8-958b-76bae5ad8d07 tempest-ServerDiagnosticsV248Test-347064190 tempest-ServerDiagnosticsV248Test-347064190-project-member] Lock "c563b7ed-4fb3-4fb0-8350-789fbe4d53a2" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 97.447s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 685.521108] env[67752]: DEBUG nova.compute.manager [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 685.573420] env[67752]: DEBUG oslo_concurrency.lockutils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 685.573669] env[67752]: DEBUG oslo_concurrency.lockutils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 685.575157] env[67752]: INFO nova.compute.claims [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 685.982656] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3d8e8f9c-4b0d-4945-9297-1c9e7e183c46 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 685.991429] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19be577e-1369-4858-9d35-89e1ff66b0b8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 686.022645] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50214862-f7e4-4cd8-9169-afd3b95ef100 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 686.030170] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b94489eb-5da7-4caf-aa69-e9c04eccb470 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 686.047690] env[67752]: DEBUG nova.compute.provider_tree [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 686.057820] env[67752]: DEBUG nova.scheduler.client.report [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 686.071836] env[67752]: DEBUG oslo_concurrency.lockutils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.498s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 686.072317] env[67752]: DEBUG nova.compute.manager [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 686.112746] env[67752]: DEBUG nova.compute.utils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 686.114119] env[67752]: DEBUG nova.compute.manager [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 686.114334] env[67752]: DEBUG nova.network.neutron [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 686.122604] env[67752]: DEBUG nova.compute.manager [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 686.191709] env[67752]: DEBUG nova.compute.manager [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 686.212548] env[67752]: DEBUG nova.policy [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'a0d363f779de4eb2a31d9b7879bb0580', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'cedc3b2e904e468eb6912ff9000b1fe6', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 686.219266] env[67752]: DEBUG nova.virt.hardware [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 686.219500] env[67752]: DEBUG nova.virt.hardware [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 686.219668] env[67752]: DEBUG nova.virt.hardware [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 686.219836] env[67752]: DEBUG nova.virt.hardware [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 686.219994] env[67752]: DEBUG nova.virt.hardware [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 686.220145] env[67752]: DEBUG nova.virt.hardware [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 686.220386] env[67752]: DEBUG nova.virt.hardware [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 686.220509] env[67752]: DEBUG nova.virt.hardware [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 686.220686] env[67752]: DEBUG nova.virt.hardware [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 686.220864] env[67752]: DEBUG nova.virt.hardware [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 686.221061] env[67752]: DEBUG nova.virt.hardware [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 686.221947] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-004c2afe-c2c8-4508-84c8-812d8225b121 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 686.229774] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e462804-bb60-4e2c-8009-94b485d0b262 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 686.904064] env[67752]: DEBUG nova.network.neutron [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Successfully created port: 5f4c26c0-61f3-4b31-a7b1-f78091cc108c {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 688.016637] env[67752]: DEBUG nova.network.neutron [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Successfully updated port: 5f4c26c0-61f3-4b31-a7b1-f78091cc108c {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 688.036260] env[67752]: DEBUG oslo_concurrency.lockutils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Acquiring lock "refresh_cache-ca18af6b-6b80-44f3-bd8b-f61e2596acd7" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 688.036408] env[67752]: DEBUG oslo_concurrency.lockutils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Acquired lock "refresh_cache-ca18af6b-6b80-44f3-bd8b-f61e2596acd7" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 688.036669] env[67752]: DEBUG nova.network.neutron [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 688.056913] env[67752]: DEBUG nova.compute.manager [req-3809b83f-92d3-4a06-952c-858271c114af req-2d8af58b-0570-4647-a643-f9571da820b9 service nova] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Received event network-vif-plugged-5f4c26c0-61f3-4b31-a7b1-f78091cc108c {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 688.057195] env[67752]: DEBUG oslo_concurrency.lockutils [req-3809b83f-92d3-4a06-952c-858271c114af req-2d8af58b-0570-4647-a643-f9571da820b9 service nova] Acquiring lock "ca18af6b-6b80-44f3-bd8b-f61e2596acd7-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 688.057371] env[67752]: DEBUG oslo_concurrency.lockutils [req-3809b83f-92d3-4a06-952c-858271c114af req-2d8af58b-0570-4647-a643-f9571da820b9 service nova] Lock "ca18af6b-6b80-44f3-bd8b-f61e2596acd7-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 688.057551] env[67752]: DEBUG oslo_concurrency.lockutils [req-3809b83f-92d3-4a06-952c-858271c114af req-2d8af58b-0570-4647-a643-f9571da820b9 service nova] Lock "ca18af6b-6b80-44f3-bd8b-f61e2596acd7-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 688.057721] env[67752]: DEBUG nova.compute.manager [req-3809b83f-92d3-4a06-952c-858271c114af req-2d8af58b-0570-4647-a643-f9571da820b9 service nova] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] No waiting events found dispatching network-vif-plugged-5f4c26c0-61f3-4b31-a7b1-f78091cc108c {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 688.057894] env[67752]: WARNING nova.compute.manager [req-3809b83f-92d3-4a06-952c-858271c114af req-2d8af58b-0570-4647-a643-f9571da820b9 service nova] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Received unexpected event network-vif-plugged-5f4c26c0-61f3-4b31-a7b1-f78091cc108c for instance with vm_state building and task_state spawning. [ 688.113424] env[67752]: DEBUG nova.network.neutron [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 688.615411] env[67752]: DEBUG nova.network.neutron [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Updating instance_info_cache with network_info: [{"id": "5f4c26c0-61f3-4b31-a7b1-f78091cc108c", "address": "fa:16:3e:8f:60:f9", "network": {"id": "0f8daa1b-9055-4a5f-b909-64b17baa3094", "bridge": "br-int", "label": "tempest-VolumesAssistedSnapshotsTest-1364428623-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "cedc3b2e904e468eb6912ff9000b1fe6", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "7041d198-66a3-40de-bf7d-cfc036e6ed69", "external-id": "nsx-vlan-transportzone-278", "segmentation_id": 278, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5f4c26c0-61", "ovs_interfaceid": "5f4c26c0-61f3-4b31-a7b1-f78091cc108c", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 688.629563] env[67752]: DEBUG oslo_concurrency.lockutils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Releasing lock "refresh_cache-ca18af6b-6b80-44f3-bd8b-f61e2596acd7" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 688.629694] env[67752]: DEBUG nova.compute.manager [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Instance network_info: |[{"id": "5f4c26c0-61f3-4b31-a7b1-f78091cc108c", "address": "fa:16:3e:8f:60:f9", "network": {"id": "0f8daa1b-9055-4a5f-b909-64b17baa3094", "bridge": "br-int", "label": "tempest-VolumesAssistedSnapshotsTest-1364428623-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "cedc3b2e904e468eb6912ff9000b1fe6", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "7041d198-66a3-40de-bf7d-cfc036e6ed69", "external-id": "nsx-vlan-transportzone-278", "segmentation_id": 278, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5f4c26c0-61", "ovs_interfaceid": "5f4c26c0-61f3-4b31-a7b1-f78091cc108c", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 688.630107] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:8f:60:f9', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '7041d198-66a3-40de-bf7d-cfc036e6ed69', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '5f4c26c0-61f3-4b31-a7b1-f78091cc108c', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 688.637587] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Creating folder: Project (cedc3b2e904e468eb6912ff9000b1fe6). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 688.638161] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-106aa958-18d6-401b-817e-0543ed4cef9c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 688.648800] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Created folder: Project (cedc3b2e904e468eb6912ff9000b1fe6) in parent group-v639722. [ 688.649011] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Creating folder: Instances. Parent ref: group-v639761. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 688.649278] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-5cc8af29-e41e-4120-8f62-0dd3da87d7c2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 688.658190] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Created folder: Instances in parent group-v639761. [ 688.658190] env[67752]: DEBUG oslo.service.loopingcall [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 688.658190] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 688.658190] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-f0c8d2f3-5828-4df3-9163-6f35e4d1c451 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 688.679365] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 688.679365] env[67752]: value = "task-3199649" [ 688.679365] env[67752]: _type = "Task" [ 688.679365] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 688.691145] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199649, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 689.188528] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199649, 'name': CreateVM_Task, 'duration_secs': 0.296306} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 689.188857] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 689.190548] env[67752]: DEBUG oslo_concurrency.lockutils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 689.190548] env[67752]: DEBUG oslo_concurrency.lockutils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 689.190548] env[67752]: DEBUG oslo_concurrency.lockutils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 689.190548] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-dc1ae011-2cab-452c-bae5-a1ad7fcb8821 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 689.194875] env[67752]: DEBUG oslo_vmware.api [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Waiting for the task: (returnval){ [ 689.194875] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52826e12-9363-bf09-93aa-563261fed3ce" [ 689.194875] env[67752]: _type = "Task" [ 689.194875] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 689.207016] env[67752]: DEBUG oslo_vmware.api [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52826e12-9363-bf09-93aa-563261fed3ce, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 689.707510] env[67752]: DEBUG oslo_concurrency.lockutils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 689.707771] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 689.708246] env[67752]: DEBUG oslo_concurrency.lockutils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 690.550411] env[67752]: DEBUG nova.compute.manager [req-e852b206-4bb5-4794-93fd-209d898ac242 req-26a657b9-bf16-486c-bacf-7bb544739291 service nova] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Received event network-changed-5f4c26c0-61f3-4b31-a7b1-f78091cc108c {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 690.550664] env[67752]: DEBUG nova.compute.manager [req-e852b206-4bb5-4794-93fd-209d898ac242 req-26a657b9-bf16-486c-bacf-7bb544739291 service nova] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Refreshing instance network info cache due to event network-changed-5f4c26c0-61f3-4b31-a7b1-f78091cc108c. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 690.550842] env[67752]: DEBUG oslo_concurrency.lockutils [req-e852b206-4bb5-4794-93fd-209d898ac242 req-26a657b9-bf16-486c-bacf-7bb544739291 service nova] Acquiring lock "refresh_cache-ca18af6b-6b80-44f3-bd8b-f61e2596acd7" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 690.550989] env[67752]: DEBUG oslo_concurrency.lockutils [req-e852b206-4bb5-4794-93fd-209d898ac242 req-26a657b9-bf16-486c-bacf-7bb544739291 service nova] Acquired lock "refresh_cache-ca18af6b-6b80-44f3-bd8b-f61e2596acd7" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 690.551169] env[67752]: DEBUG nova.network.neutron [req-e852b206-4bb5-4794-93fd-209d898ac242 req-26a657b9-bf16-486c-bacf-7bb544739291 service nova] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Refreshing network info cache for port 5f4c26c0-61f3-4b31-a7b1-f78091cc108c {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 691.287452] env[67752]: DEBUG nova.network.neutron [req-e852b206-4bb5-4794-93fd-209d898ac242 req-26a657b9-bf16-486c-bacf-7bb544739291 service nova] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Updated VIF entry in instance network info cache for port 5f4c26c0-61f3-4b31-a7b1-f78091cc108c. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 691.287937] env[67752]: DEBUG nova.network.neutron [req-e852b206-4bb5-4794-93fd-209d898ac242 req-26a657b9-bf16-486c-bacf-7bb544739291 service nova] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Updating instance_info_cache with network_info: [{"id": "5f4c26c0-61f3-4b31-a7b1-f78091cc108c", "address": "fa:16:3e:8f:60:f9", "network": {"id": "0f8daa1b-9055-4a5f-b909-64b17baa3094", "bridge": "br-int", "label": "tempest-VolumesAssistedSnapshotsTest-1364428623-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "cedc3b2e904e468eb6912ff9000b1fe6", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "7041d198-66a3-40de-bf7d-cfc036e6ed69", "external-id": "nsx-vlan-transportzone-278", "segmentation_id": 278, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5f4c26c0-61", "ovs_interfaceid": "5f4c26c0-61f3-4b31-a7b1-f78091cc108c", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 691.302555] env[67752]: DEBUG oslo_concurrency.lockutils [req-e852b206-4bb5-4794-93fd-209d898ac242 req-26a657b9-bf16-486c-bacf-7bb544739291 service nova] Releasing lock "refresh_cache-ca18af6b-6b80-44f3-bd8b-f61e2596acd7" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 692.837412] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Acquiring lock "411bafb7-ef72-4529-b3f3-0b4955f23788" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 692.837412] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Lock "411bafb7-ef72-4529-b3f3-0b4955f23788" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 711.635632] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 711.635890] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 711.635890] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 711.656920] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 711.657088] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 711.657225] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 711.657356] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 711.657482] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 711.657610] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 711.657717] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 711.657841] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 711.657951] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 711.658080] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 711.658207] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 713.634301] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 713.634614] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 713.634684] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 713.634838] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 713.634991] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 713.635153] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 713.647581] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 713.647804] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 713.647975] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 713.648180] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 713.649466] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-07063fc9-a2df-464f-8832-7b3316d5850a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 713.658151] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-37236827-8743-4d83-82df-d76c035a6569 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 713.671472] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9e41f171-2f3c-4bba-8746-2aa6acf22d00 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 713.677435] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec353519-ab59-4cd4-bf2a-c06a27de2924 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 713.705294] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181011MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 713.705436] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 713.705623] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 713.782594] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 713.782755] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fc4da5fa-1921-44e3-9100-042391d4ccb9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 713.782882] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b5eee51b-73b8-4e17-98a4-e108a5b21892 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 713.783013] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 39479042-b891-4f25-9992-47312c6b6b43 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 713.783141] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 713.783255] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 713.783370] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b0cc1acd-0e33-4547-a43b-549b3de81728 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 713.783489] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance eca7c94f-9bd1-4c15-988e-d4e33086ba40 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 713.783626] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ff4895f8-0374-4b34-a5e8-d3ba24f840a5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 713.783743] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ca18af6b-6b80-44f3-bd8b-f61e2596acd7 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 713.795134] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance cbe3fe72-5a91-4c81-ac62-be6592aa806a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 713.809037] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 02942b29-2343-441e-9e2b-aca2af57c849 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 713.820564] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ebd6c719-32ae-410c-a444-db7ad3f8a9c5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 713.830476] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f528dc15-1a86-4af5-ac45-536fe3dfe101 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 713.839651] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 4c0873a7-b8e5-4d3e-bdc0-3d236e3b9df6 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 713.849189] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e3c2e1cf-cae5-474f-9f61-96d79815d846 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 713.860499] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f22bcbae-0dcb-4666-b5e7-00c560e016c3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 713.870756] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c38268b5-b35d-4d8d-aa03-72100e2889f7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 713.881148] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c6a7f89e-d842-4b34-af36-ca60f12bcddd has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 713.891096] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fa9cdf6d-032f-4069-a78e-fb16b4d2c457 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 713.900903] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 713.910893] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 40aaf487-275a-4e75-ac6a-127371e73b2c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 713.920714] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d2104921-5ec1-4fb8-8034-52dcdc403a92 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 713.931134] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 16f0124d-146a-442a-8738-65f562b928e8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 713.941341] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 8503063e-2888-44c6-bc2a-e9c82be791e2 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 713.951554] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 631002d2-c268-4a6a-9945-c0898b886b93 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 713.962416] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 67812e2e-8827-4368-9790-a909c5ebb56b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 713.975218] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 5ed08031-0ef8-4766-951a-37b315c55dc7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 713.984859] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 436d243a-d6d9-4958-94a8-7b09f5d9fdf3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 713.994808] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f8b03251-6f23-4533-a917-f3fdac87888b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 714.003720] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 58d3b02f-3aac-4beb-a111-44f50c7f1bc8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 714.013030] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 53eb8b0c-aa50-4d7f-8027-2725c815f134 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 714.021907] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 411bafb7-ef72-4529-b3f3-0b4955f23788 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 714.022155] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 714.022338] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 714.401653] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3efd2e68-01e7-4c42-b041-25acdd60642d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.413457] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ca120204-2a85-49bf-81bd-fd2a71644470 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.465755] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-818b1952-ea18-4154-ac5e-93adb9e82ba5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.477068] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7beeae1-4456-48ad-a7a6-fe1bdf3c46ed {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 714.498472] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 714.509786] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 714.528360] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 714.528601] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.823s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 715.528683] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 715.528991] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 715.635247] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 733.422045] env[67752]: WARNING oslo_vmware.rw_handles [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 733.422045] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 733.422045] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 733.422045] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 733.422045] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 733.422045] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 733.422045] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 733.422045] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 733.422045] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 733.422045] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 733.422045] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 733.422045] env[67752]: ERROR oslo_vmware.rw_handles [ 733.422045] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/457c77fd-4e5d-4e92-9ba7-602bedf0e43f/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 733.424208] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 733.424500] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Copying Virtual Disk [datastore2] vmware_temp/457c77fd-4e5d-4e92-9ba7-602bedf0e43f/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/457c77fd-4e5d-4e92-9ba7-602bedf0e43f/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 733.425227] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-d37c28c9-64e4-46dc-ab55-8a4da20dd8d0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 733.432751] env[67752]: DEBUG oslo_vmware.api [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Waiting for the task: (returnval){ [ 733.432751] env[67752]: value = "task-3199650" [ 733.432751] env[67752]: _type = "Task" [ 733.432751] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 733.440942] env[67752]: DEBUG oslo_vmware.api [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Task: {'id': task-3199650, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 733.943277] env[67752]: DEBUG oslo_vmware.exceptions [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 733.943579] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 733.944154] env[67752]: ERROR nova.compute.manager [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 733.944154] env[67752]: Faults: ['InvalidArgument'] [ 733.944154] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Traceback (most recent call last): [ 733.944154] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 733.944154] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] yield resources [ 733.944154] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 733.944154] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] self.driver.spawn(context, instance, image_meta, [ 733.944154] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 733.944154] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] self._vmops.spawn(context, instance, image_meta, injected_files, [ 733.944154] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 733.944154] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] self._fetch_image_if_missing(context, vi) [ 733.944154] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 733.944668] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] image_cache(vi, tmp_image_ds_loc) [ 733.944668] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 733.944668] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] vm_util.copy_virtual_disk( [ 733.944668] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 733.944668] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] session._wait_for_task(vmdk_copy_task) [ 733.944668] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 733.944668] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] return self.wait_for_task(task_ref) [ 733.944668] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 733.944668] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] return evt.wait() [ 733.944668] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 733.944668] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] result = hub.switch() [ 733.944668] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 733.944668] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] return self.greenlet.switch() [ 733.944955] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 733.944955] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] self.f(*self.args, **self.kw) [ 733.944955] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 733.944955] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] raise exceptions.translate_fault(task_info.error) [ 733.944955] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 733.944955] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Faults: ['InvalidArgument'] [ 733.944955] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] [ 733.944955] env[67752]: INFO nova.compute.manager [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Terminating instance [ 733.946098] env[67752]: DEBUG oslo_concurrency.lockutils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 733.946303] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 733.946822] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Acquiring lock "refresh_cache-b5eee51b-73b8-4e17-98a4-e108a5b21892" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 733.946975] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Acquired lock "refresh_cache-b5eee51b-73b8-4e17-98a4-e108a5b21892" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 733.947159] env[67752]: DEBUG nova.network.neutron [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 733.948096] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-7b4de4d4-9d81-402d-bf4b-4b9dcc2e2d39 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 733.957803] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 733.957985] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 733.960900] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-5348580a-da95-4176-988d-94c31bb84624 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 733.967132] env[67752]: DEBUG oslo_vmware.api [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Waiting for the task: (returnval){ [ 733.967132] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52e00b36-3639-133e-7820-112488613300" [ 733.967132] env[67752]: _type = "Task" [ 733.967132] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 733.974459] env[67752]: DEBUG oslo_vmware.api [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52e00b36-3639-133e-7820-112488613300, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 733.995063] env[67752]: DEBUG nova.network.neutron [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 734.097578] env[67752]: DEBUG nova.network.neutron [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 734.108386] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Releasing lock "refresh_cache-b5eee51b-73b8-4e17-98a4-e108a5b21892" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 734.108906] env[67752]: DEBUG nova.compute.manager [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 734.109114] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 734.110298] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf658abf-46f6-4c20-84a0-ec753701066d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 734.118717] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 734.119142] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-86ecbe7d-dac1-41ab-a899-5765a6db07eb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 734.154867] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 734.155114] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 734.155298] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Deleting the datastore file [datastore2] b5eee51b-73b8-4e17-98a4-e108a5b21892 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 734.155544] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-edb95868-afcd-42c1-8e82-5951015428a6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 734.162047] env[67752]: DEBUG oslo_vmware.api [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Waiting for the task: (returnval){ [ 734.162047] env[67752]: value = "task-3199652" [ 734.162047] env[67752]: _type = "Task" [ 734.162047] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 734.170685] env[67752]: DEBUG oslo_vmware.api [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Task: {'id': task-3199652, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 734.478321] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 734.478618] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Creating directory with path [datastore2] vmware_temp/2ceff526-cc04-486a-928b-7d151165f80c/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 734.478778] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-2da4fe39-56b4-48d6-a6f8-b59faa249f27 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 734.490373] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Created directory with path [datastore2] vmware_temp/2ceff526-cc04-486a-928b-7d151165f80c/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 734.490573] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Fetch image to [datastore2] vmware_temp/2ceff526-cc04-486a-928b-7d151165f80c/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 734.490744] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/2ceff526-cc04-486a-928b-7d151165f80c/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 734.491495] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-485607e2-5e9b-41d1-b279-fc66d4e66151 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 734.498164] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9f2d21d-06ce-459f-84e9-723c07185620 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 734.506927] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c07b6c9a-05f8-4efb-8c08-8d4f62b32e39 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 734.537276] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ccd8d90e-5733-4bde-bc84-c9432d596c51 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 734.543013] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-85100ab2-ac71-4ad1-8ac4-670dc439050d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 734.566682] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 734.618355] env[67752]: DEBUG oslo_vmware.rw_handles [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/2ceff526-cc04-486a-928b-7d151165f80c/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 734.680780] env[67752]: DEBUG oslo_vmware.rw_handles [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 734.681011] env[67752]: DEBUG oslo_vmware.rw_handles [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Closing write handle for https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/2ceff526-cc04-486a-928b-7d151165f80c/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 734.684992] env[67752]: DEBUG oslo_vmware.api [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Task: {'id': task-3199652, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.031244} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 734.685241] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 734.685423] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 734.685598] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 734.685774] env[67752]: INFO nova.compute.manager [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Took 0.58 seconds to destroy the instance on the hypervisor. [ 734.686015] env[67752]: DEBUG oslo.service.loopingcall [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 734.686224] env[67752]: DEBUG nova.compute.manager [-] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Skipping network deallocation for instance since networking was not requested. {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2276}} [ 734.688459] env[67752]: DEBUG nova.compute.claims [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 734.688632] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 734.688861] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.092292] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56e4bd11-0924-4efb-b958-7a741352cb18 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 735.100923] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-163c1cbd-e30c-41bd-adcf-7f075042474e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 735.133235] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86d9edb5-4224-47ae-99ca-aa7ab4f42906 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 735.140545] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ec956f7-6914-41e0-a2b3-16cf152cac06 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 735.154197] env[67752]: DEBUG nova.compute.provider_tree [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 735.162974] env[67752]: DEBUG nova.scheduler.client.report [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 735.177220] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.488s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.177748] env[67752]: ERROR nova.compute.manager [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 735.177748] env[67752]: Faults: ['InvalidArgument'] [ 735.177748] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Traceback (most recent call last): [ 735.177748] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 735.177748] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] self.driver.spawn(context, instance, image_meta, [ 735.177748] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 735.177748] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] self._vmops.spawn(context, instance, image_meta, injected_files, [ 735.177748] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 735.177748] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] self._fetch_image_if_missing(context, vi) [ 735.177748] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 735.177748] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] image_cache(vi, tmp_image_ds_loc) [ 735.177748] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 735.178051] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] vm_util.copy_virtual_disk( [ 735.178051] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 735.178051] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] session._wait_for_task(vmdk_copy_task) [ 735.178051] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 735.178051] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] return self.wait_for_task(task_ref) [ 735.178051] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 735.178051] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] return evt.wait() [ 735.178051] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 735.178051] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] result = hub.switch() [ 735.178051] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 735.178051] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] return self.greenlet.switch() [ 735.178051] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 735.178051] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] self.f(*self.args, **self.kw) [ 735.178333] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 735.178333] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] raise exceptions.translate_fault(task_info.error) [ 735.178333] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 735.178333] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Faults: ['InvalidArgument'] [ 735.178333] env[67752]: ERROR nova.compute.manager [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] [ 735.178449] env[67752]: DEBUG nova.compute.utils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 735.179887] env[67752]: DEBUG nova.compute.manager [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Build of instance b5eee51b-73b8-4e17-98a4-e108a5b21892 was re-scheduled: A specified parameter was not correct: fileType [ 735.179887] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 735.180303] env[67752]: DEBUG nova.compute.manager [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 735.180526] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Acquiring lock "refresh_cache-b5eee51b-73b8-4e17-98a4-e108a5b21892" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 735.180673] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Acquired lock "refresh_cache-b5eee51b-73b8-4e17-98a4-e108a5b21892" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 735.180833] env[67752]: DEBUG nova.network.neutron [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 735.206572] env[67752]: DEBUG nova.network.neutron [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 735.325814] env[67752]: DEBUG nova.network.neutron [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 735.334473] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Releasing lock "refresh_cache-b5eee51b-73b8-4e17-98a4-e108a5b21892" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 735.334688] env[67752]: DEBUG nova.compute.manager [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 735.334880] env[67752]: DEBUG nova.compute.manager [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] [instance: b5eee51b-73b8-4e17-98a4-e108a5b21892] Skipping network deallocation for instance since networking was not requested. {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2276}} [ 735.429166] env[67752]: INFO nova.scheduler.client.report [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Deleted allocations for instance b5eee51b-73b8-4e17-98a4-e108a5b21892 [ 735.452280] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1fe2ddef-1f8c-4ed1-bbba-383645036f15 tempest-ServersAdmin275Test-1019243502 tempest-ServersAdmin275Test-1019243502-project-member] Lock "b5eee51b-73b8-4e17-98a4-e108a5b21892" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 146.670s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.466020] env[67752]: DEBUG nova.compute.manager [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 735.518695] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.518973] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.520574] env[67752]: INFO nova.compute.claims [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 735.955393] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-242be7ac-24e4-4542-a0f8-17b38998aa8e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 735.964310] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dee78278-d548-4a14-a9a5-89a12927edac {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 735.993368] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a39d4c9-f37a-4366-ab9e-e6d47668ddb9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 736.000422] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2679bd71-3a7e-460c-911a-e0aea6a54bf8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 736.013391] env[67752]: DEBUG nova.compute.provider_tree [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 736.022538] env[67752]: DEBUG nova.scheduler.client.report [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 736.041108] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.522s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 736.041501] env[67752]: DEBUG nova.compute.manager [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 736.089344] env[67752]: DEBUG nova.compute.utils [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 736.090677] env[67752]: DEBUG nova.compute.manager [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 736.090862] env[67752]: DEBUG nova.network.neutron [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 736.101520] env[67752]: DEBUG nova.compute.manager [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 736.136677] env[67752]: INFO nova.virt.block_device [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Booting with volume 36021ac9-93ea-424f-9037-401e63e60004 at /dev/sda [ 736.188125] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-53269e95-c9cc-40b1-b945-b76d930dbe84 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 736.197532] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-213e7ff9-06e3-4763-8fdb-8a48b07203c2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 736.225615] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c16a4983-0518-4f1e-8aa9-4b8dc38382a1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 736.233163] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59bb03a5-51ae-4d7d-a9f2-277c7521f3cf {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 736.261116] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e60e841d-3989-400a-b958-b43e577dea75 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 736.267138] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d7c0782-5119-477a-b533-62d507899b08 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 736.279914] env[67752]: DEBUG nova.virt.block_device [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Updating existing volume attachment record: dbeefc9b-2a02-4d06-9cc4-797c516a52c4 {{(pid=67752) _volume_attach /opt/stack/nova/nova/virt/block_device.py:665}} [ 736.291194] env[67752]: DEBUG nova.policy [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'fd2a4119a94a48528e07fb99f3d20d42', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'c1a16a3476764b939f260917e4b7391a', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 736.522565] env[67752]: DEBUG nova.compute.manager [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 736.523111] env[67752]: DEBUG nova.virt.hardware [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum=,container_format=,created_at=,direct_url=,disk_format=,id=,min_disk=0,min_ram=0,name=,owner=,properties=ImageMetaProps,protected=,size=1073741824,status='active',tags=,updated_at=,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 736.523341] env[67752]: DEBUG nova.virt.hardware [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 736.523586] env[67752]: DEBUG nova.virt.hardware [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 736.523689] env[67752]: DEBUG nova.virt.hardware [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 736.523902] env[67752]: DEBUG nova.virt.hardware [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 736.524430] env[67752]: DEBUG nova.virt.hardware [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 736.524697] env[67752]: DEBUG nova.virt.hardware [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 736.525072] env[67752]: DEBUG nova.virt.hardware [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 736.525072] env[67752]: DEBUG nova.virt.hardware [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 736.525244] env[67752]: DEBUG nova.virt.hardware [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 736.525420] env[67752]: DEBUG nova.virt.hardware [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 736.527079] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-26dc5c2c-1cff-4b0f-99ce-da29b92e487f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 736.536206] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1c308c0-51f2-41f4-95fe-290c24bc482b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 736.982244] env[67752]: DEBUG nova.network.neutron [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Successfully created port: 6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 738.399284] env[67752]: DEBUG nova.network.neutron [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Successfully updated port: 6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 738.420070] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Acquiring lock "refresh_cache-cbe3fe72-5a91-4c81-ac62-be6592aa806a" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 738.420228] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Acquired lock "refresh_cache-cbe3fe72-5a91-4c81-ac62-be6592aa806a" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 738.420379] env[67752]: DEBUG nova.network.neutron [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 738.503940] env[67752]: DEBUG nova.network.neutron [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 738.596959] env[67752]: DEBUG nova.compute.manager [req-41d08e84-7cae-49fa-8dc0-7a2276b92aaa req-9f312306-4762-46e7-b9ab-4c8cb03fd536 service nova] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Received event network-vif-plugged-6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 738.597044] env[67752]: DEBUG oslo_concurrency.lockutils [req-41d08e84-7cae-49fa-8dc0-7a2276b92aaa req-9f312306-4762-46e7-b9ab-4c8cb03fd536 service nova] Acquiring lock "cbe3fe72-5a91-4c81-ac62-be6592aa806a-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 738.597232] env[67752]: DEBUG oslo_concurrency.lockutils [req-41d08e84-7cae-49fa-8dc0-7a2276b92aaa req-9f312306-4762-46e7-b9ab-4c8cb03fd536 service nova] Lock "cbe3fe72-5a91-4c81-ac62-be6592aa806a-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 738.599089] env[67752]: DEBUG oslo_concurrency.lockutils [req-41d08e84-7cae-49fa-8dc0-7a2276b92aaa req-9f312306-4762-46e7-b9ab-4c8cb03fd536 service nova] Lock "cbe3fe72-5a91-4c81-ac62-be6592aa806a-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 738.599089] env[67752]: DEBUG nova.compute.manager [req-41d08e84-7cae-49fa-8dc0-7a2276b92aaa req-9f312306-4762-46e7-b9ab-4c8cb03fd536 service nova] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] No waiting events found dispatching network-vif-plugged-6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 738.599089] env[67752]: WARNING nova.compute.manager [req-41d08e84-7cae-49fa-8dc0-7a2276b92aaa req-9f312306-4762-46e7-b9ab-4c8cb03fd536 service nova] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Received unexpected event network-vif-plugged-6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2 for instance with vm_state building and task_state spawning. [ 738.799700] env[67752]: DEBUG nova.network.neutron [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Updating instance_info_cache with network_info: [{"id": "6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2", "address": "fa:16:3e:97:43:ec", "network": {"id": "c8fa9bf0-50df-4d49-9e40-03d925f235f2", "bridge": "br-int", "label": "tempest-ServerActionsV293TestJSON-2009741725-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "c1a16a3476764b939f260917e4b7391a", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a4fc0575-c8e3-4f3c-b2e1-e10ac2d0cc1a", "external-id": "nsx-vlan-transportzone-256", "segmentation_id": 256, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap6fdef7ca-a3", "ovs_interfaceid": "6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 738.818645] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Releasing lock "refresh_cache-cbe3fe72-5a91-4c81-ac62-be6592aa806a" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 738.818645] env[67752]: DEBUG nova.compute.manager [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Instance network_info: |[{"id": "6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2", "address": "fa:16:3e:97:43:ec", "network": {"id": "c8fa9bf0-50df-4d49-9e40-03d925f235f2", "bridge": "br-int", "label": "tempest-ServerActionsV293TestJSON-2009741725-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "c1a16a3476764b939f260917e4b7391a", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a4fc0575-c8e3-4f3c-b2e1-e10ac2d0cc1a", "external-id": "nsx-vlan-transportzone-256", "segmentation_id": 256, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap6fdef7ca-a3", "ovs_interfaceid": "6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 738.818783] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:97:43:ec', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'a4fc0575-c8e3-4f3c-b2e1-e10ac2d0cc1a', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 738.829029] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Creating folder: Project (c1a16a3476764b939f260917e4b7391a). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 738.829156] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-1e9d61c0-eb45-4db1-980b-e03c08327313 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 738.844551] env[67752]: WARNING suds.client [-] Web service reported a SOAP processing fault using an unexpected HTTP status code 200. Reporting as an internal server error. [ 738.845170] env[67752]: DEBUG oslo_vmware.api [-] Fault list: [DuplicateName] {{(pid=67752) _invoke_api /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:337}} [ 738.845305] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Folder already exists: Project (c1a16a3476764b939f260917e4b7391a). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 738.845562] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Creating folder: Instances. Parent ref: group-v639726. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 738.845721] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-9889591f-0513-4a8b-89fc-9b2c0cc5f5ae {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 738.858018] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Created folder: Instances in parent group-v639726. [ 738.858018] env[67752]: DEBUG oslo.service.loopingcall [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 738.858018] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 738.858018] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-e7908107-b01b-4cce-8b14-b4c7eeafa9f0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 738.877298] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 738.877298] env[67752]: value = "task-3199655" [ 738.877298] env[67752]: _type = "Task" [ 738.877298] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 738.890405] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199655, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 739.389892] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199655, 'name': CreateVM_Task, 'duration_secs': 0.349882} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 739.390258] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 739.391096] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Block device information present: {'root_device_name': '/dev/sda', 'image': [], 'ephemerals': [], 'block_device_mapping': [{'connection_info': {'driver_volume_type': 'vmdk', 'data': {'volume': 'vm-639741', 'volume_id': '36021ac9-93ea-424f-9037-401e63e60004', 'name': 'volume-36021ac9-93ea-424f-9037-401e63e60004', 'profile_id': None, 'qos_specs': None, 'access_mode': 'rw', 'encrypted': False, 'cacheable': False}, 'status': 'reserved', 'instance': 'cbe3fe72-5a91-4c81-ac62-be6592aa806a', 'attached_at': '', 'detached_at': '', 'volume_id': '36021ac9-93ea-424f-9037-401e63e60004', 'serial': '36021ac9-93ea-424f-9037-401e63e60004'}, 'delete_on_termination': True, 'guest_format': None, 'attachment_id': 'dbeefc9b-2a02-4d06-9cc4-797c516a52c4', 'disk_bus': None, 'boot_index': 0, 'device_type': None, 'mount_device': '/dev/sda', 'volume_type': None}], 'swap': None} {{(pid=67752) spawn /opt/stack/nova/nova/virt/vmwareapi/vmops.py:799}} [ 739.391545] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Root volume attach. Driver type: vmdk {{(pid=67752) attach_root_volume /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:661}} [ 739.392781] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7fafa7b0-aded-4be7-9869-dcd4ee3d272d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 739.404056] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-80976568-8cd3-4c73-af5a-2ecce39efcb2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 739.412316] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0c11690e-ce9a-4506-9592-aff13818df42 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 739.420738] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.RelocateVM_Task with opID=oslo.vmware-94a550a2-2d72-4fe2-9179-70878aadf671 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 739.428774] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Waiting for the task: (returnval){ [ 739.428774] env[67752]: value = "task-3199656" [ 739.428774] env[67752]: _type = "Task" [ 739.428774] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 739.438846] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199656, 'name': RelocateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 739.940023] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199656, 'name': RelocateVM_Task} progress is 40%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 740.441361] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199656, 'name': RelocateVM_Task} progress is 53%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 740.948558] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199656, 'name': RelocateVM_Task} progress is 65%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 741.333901] env[67752]: DEBUG nova.compute.manager [req-6e7077ad-e4b9-4a7a-a0ad-b2c17bf453fe req-b7f2116b-e860-4566-826c-b637ed024abf service nova] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Received event network-changed-6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 741.334209] env[67752]: DEBUG nova.compute.manager [req-6e7077ad-e4b9-4a7a-a0ad-b2c17bf453fe req-b7f2116b-e860-4566-826c-b637ed024abf service nova] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Refreshing instance network info cache due to event network-changed-6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 741.334486] env[67752]: DEBUG oslo_concurrency.lockutils [req-6e7077ad-e4b9-4a7a-a0ad-b2c17bf453fe req-b7f2116b-e860-4566-826c-b637ed024abf service nova] Acquiring lock "refresh_cache-cbe3fe72-5a91-4c81-ac62-be6592aa806a" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 741.334640] env[67752]: DEBUG oslo_concurrency.lockutils [req-6e7077ad-e4b9-4a7a-a0ad-b2c17bf453fe req-b7f2116b-e860-4566-826c-b637ed024abf service nova] Acquired lock "refresh_cache-cbe3fe72-5a91-4c81-ac62-be6592aa806a" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 741.334804] env[67752]: DEBUG nova.network.neutron [req-6e7077ad-e4b9-4a7a-a0ad-b2c17bf453fe req-b7f2116b-e860-4566-826c-b637ed024abf service nova] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Refreshing network info cache for port 6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 741.446249] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199656, 'name': RelocateVM_Task} progress is 81%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 741.945757] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199656, 'name': RelocateVM_Task} progress is 95%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 741.959249] env[67752]: DEBUG nova.network.neutron [req-6e7077ad-e4b9-4a7a-a0ad-b2c17bf453fe req-b7f2116b-e860-4566-826c-b637ed024abf service nova] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Updated VIF entry in instance network info cache for port 6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 741.959249] env[67752]: DEBUG nova.network.neutron [req-6e7077ad-e4b9-4a7a-a0ad-b2c17bf453fe req-b7f2116b-e860-4566-826c-b637ed024abf service nova] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Updating instance_info_cache with network_info: [{"id": "6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2", "address": "fa:16:3e:97:43:ec", "network": {"id": "c8fa9bf0-50df-4d49-9e40-03d925f235f2", "bridge": "br-int", "label": "tempest-ServerActionsV293TestJSON-2009741725-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "c1a16a3476764b939f260917e4b7391a", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a4fc0575-c8e3-4f3c-b2e1-e10ac2d0cc1a", "external-id": "nsx-vlan-transportzone-256", "segmentation_id": 256, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap6fdef7ca-a3", "ovs_interfaceid": "6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 741.968565] env[67752]: DEBUG oslo_concurrency.lockutils [req-6e7077ad-e4b9-4a7a-a0ad-b2c17bf453fe req-b7f2116b-e860-4566-826c-b637ed024abf service nova] Releasing lock "refresh_cache-cbe3fe72-5a91-4c81-ac62-be6592aa806a" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 742.446930] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199656, 'name': RelocateVM_Task} progress is 97%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 742.947373] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199656, 'name': RelocateVM_Task, 'duration_secs': 3.35014} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 742.949148] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Volume attach. Driver type: vmdk {{(pid=67752) attach_volume /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:439}} [ 742.949148] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] _attach_volume_vmdk: {'driver_volume_type': 'vmdk', 'data': {'volume': 'vm-639741', 'volume_id': '36021ac9-93ea-424f-9037-401e63e60004', 'name': 'volume-36021ac9-93ea-424f-9037-401e63e60004', 'profile_id': None, 'qos_specs': None, 'access_mode': 'rw', 'encrypted': False, 'cacheable': False}, 'status': 'reserved', 'instance': 'cbe3fe72-5a91-4c81-ac62-be6592aa806a', 'attached_at': '', 'detached_at': '', 'volume_id': '36021ac9-93ea-424f-9037-401e63e60004', 'serial': '36021ac9-93ea-424f-9037-401e63e60004'} {{(pid=67752) _attach_volume_vmdk /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:336}} [ 742.949148] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-26606955-0bbc-4d2b-a37e-324a06f1e18d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 742.968719] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b5825f19-1894-405f-8fec-cd27ae78e93c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 742.992321] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Reconfiguring VM instance instance-0000000e to attach disk [datastore2] volume-36021ac9-93ea-424f-9037-401e63e60004/volume-36021ac9-93ea-424f-9037-401e63e60004.vmdk or device None with type thin {{(pid=67752) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 742.992665] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-7123c0dc-a566-4bbe-94bc-c1a0bb5d0479 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 743.014965] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Waiting for the task: (returnval){ [ 743.014965] env[67752]: value = "task-3199657" [ 743.014965] env[67752]: _type = "Task" [ 743.014965] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 743.022857] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199657, 'name': ReconfigVM_Task} progress is 6%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 743.046328] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Acquiring lock "ada70883-9bef-496e-a65f-ca84f6ee6211" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 743.046328] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Lock "ada70883-9bef-496e-a65f-ca84f6ee6211" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 743.525728] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199657, 'name': ReconfigVM_Task} progress is 14%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 744.026011] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199657, 'name': ReconfigVM_Task, 'duration_secs': 0.79209} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 744.026263] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Reconfigured VM instance instance-0000000e to attach disk [datastore2] volume-36021ac9-93ea-424f-9037-401e63e60004/volume-36021ac9-93ea-424f-9037-401e63e60004.vmdk or device None with type thin {{(pid=67752) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 744.030907] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-c88f3200-0780-4d77-a035-eecf442850eb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 744.046827] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Waiting for the task: (returnval){ [ 744.046827] env[67752]: value = "task-3199658" [ 744.046827] env[67752]: _type = "Task" [ 744.046827] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 744.056809] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199658, 'name': ReconfigVM_Task} progress is 10%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 744.557350] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199658, 'name': ReconfigVM_Task, 'duration_secs': 0.152066} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 744.558206] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Attached VMDK: {'driver_volume_type': 'vmdk', 'data': {'volume': 'vm-639741', 'volume_id': '36021ac9-93ea-424f-9037-401e63e60004', 'name': 'volume-36021ac9-93ea-424f-9037-401e63e60004', 'profile_id': None, 'qos_specs': None, 'access_mode': 'rw', 'encrypted': False, 'cacheable': False}, 'status': 'reserved', 'instance': 'cbe3fe72-5a91-4c81-ac62-be6592aa806a', 'attached_at': '', 'detached_at': '', 'volume_id': '36021ac9-93ea-424f-9037-401e63e60004', 'serial': '36021ac9-93ea-424f-9037-401e63e60004'} {{(pid=67752) _attach_volume_vmdk /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:361}} [ 744.559049] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-a7107bad-e091-4f49-94bd-5a2320f89e45 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 744.565398] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Waiting for the task: (returnval){ [ 744.565398] env[67752]: value = "task-3199659" [ 744.565398] env[67752]: _type = "Task" [ 744.565398] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 744.573013] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199659, 'name': Rename_Task} progress is 5%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 745.075428] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199659, 'name': Rename_Task, 'duration_secs': 0.12557} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 745.075700] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Powering on the VM {{(pid=67752) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 745.075933] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-3b322d47-1969-42ab-9643-435206a939c9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 745.082204] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Waiting for the task: (returnval){ [ 745.082204] env[67752]: value = "task-3199660" [ 745.082204] env[67752]: _type = "Task" [ 745.082204] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 745.089527] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199660, 'name': PowerOnVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 745.591800] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199660, 'name': PowerOnVM_Task} progress is 66%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 746.092854] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199660, 'name': PowerOnVM_Task} progress is 66%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 746.594184] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199660, 'name': PowerOnVM_Task} progress is 66%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 747.094760] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199660, 'name': PowerOnVM_Task} progress is 66%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 747.596264] env[67752]: DEBUG oslo_vmware.api [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199660, 'name': PowerOnVM_Task, 'duration_secs': 2.290359} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 747.596627] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Powered on the VM {{(pid=67752) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 747.596816] env[67752]: INFO nova.compute.manager [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Took 11.07 seconds to spawn the instance on the hypervisor. [ 747.597036] env[67752]: DEBUG nova.compute.manager [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Checking state {{(pid=67752) _get_power_state /opt/stack/nova/nova/compute/manager.py:1782}} [ 747.597832] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1080d03-0af0-4712-8b78-02c519510e6d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 747.653103] env[67752]: INFO nova.compute.manager [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Took 12.15 seconds to build instance. [ 747.668247] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4e4c1ad9-6be9-4685-a767-732438e2683d tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Lock "cbe3fe72-5a91-4c81-ac62-be6592aa806a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 136.437s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 747.682904] env[67752]: DEBUG nova.compute.manager [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 747.739690] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 747.739762] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 747.741497] env[67752]: INFO nova.compute.claims [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 748.205954] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79bf7cfe-d232-481c-a96c-c9445793c598 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.214257] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7b80544d-8e1f-442f-bb88-fd4bb1d0e99e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.246194] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1329811c-6f20-477f-9523-c6f216048686 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.253676] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c1182443-0c37-4d59-9a7d-225aeba828ce {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.267408] env[67752]: DEBUG nova.compute.provider_tree [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 748.278666] env[67752]: DEBUG nova.scheduler.client.report [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 748.296178] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.556s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 748.296680] env[67752]: DEBUG nova.compute.manager [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 748.338404] env[67752]: DEBUG nova.compute.utils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 748.342267] env[67752]: DEBUG nova.compute.manager [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 748.342267] env[67752]: DEBUG nova.network.neutron [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 748.358845] env[67752]: DEBUG nova.compute.manager [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 748.439273] env[67752]: DEBUG nova.compute.manager [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 748.460828] env[67752]: DEBUG nova.policy [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '4bb5d2f11d864089828671348c7c1701', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd788eadbca324d08bc7afcac518d6585', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 748.473406] env[67752]: DEBUG nova.virt.hardware [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 748.473661] env[67752]: DEBUG nova.virt.hardware [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 748.473818] env[67752]: DEBUG nova.virt.hardware [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 748.474007] env[67752]: DEBUG nova.virt.hardware [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 748.474199] env[67752]: DEBUG nova.virt.hardware [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 748.474352] env[67752]: DEBUG nova.virt.hardware [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 748.474701] env[67752]: DEBUG nova.virt.hardware [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 748.475532] env[67752]: DEBUG nova.virt.hardware [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 748.475532] env[67752]: DEBUG nova.virt.hardware [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 748.475532] env[67752]: DEBUG nova.virt.hardware [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 748.475532] env[67752]: DEBUG nova.virt.hardware [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 748.476719] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd066599-5cc0-437b-9df4-19ef4c6bb86a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 748.485038] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4af7e54-9c4f-4e05-a5df-8bacf656b79d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 749.124774] env[67752]: DEBUG nova.network.neutron [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Successfully created port: 06938dfa-9e35-4180-b0e3-9eea613de885 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 751.245074] env[67752]: DEBUG nova.network.neutron [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Successfully updated port: 06938dfa-9e35-4180-b0e3-9eea613de885 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 751.255456] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Acquiring lock "refresh_cache-02942b29-2343-441e-9e2b-aca2af57c849" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 751.255604] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Acquired lock "refresh_cache-02942b29-2343-441e-9e2b-aca2af57c849" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 751.255755] env[67752]: DEBUG nova.network.neutron [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 751.334642] env[67752]: DEBUG nova.network.neutron [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 751.649814] env[67752]: DEBUG nova.network.neutron [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Updating instance_info_cache with network_info: [{"id": "06938dfa-9e35-4180-b0e3-9eea613de885", "address": "fa:16:3e:2a:1c:0a", "network": {"id": "6518e147-9c01-4e6f-b77b-4721f647fccf", "bridge": "br-int", "label": "tempest-ServersTestJSON-48688194-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d788eadbca324d08bc7afcac518d6585", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "310b8ba9-edca-4135-863e-f4a786dd4a77", "external-id": "nsx-vlan-transportzone-768", "segmentation_id": 768, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap06938dfa-9e", "ovs_interfaceid": "06938dfa-9e35-4180-b0e3-9eea613de885", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 751.660267] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Releasing lock "refresh_cache-02942b29-2343-441e-9e2b-aca2af57c849" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 751.660547] env[67752]: DEBUG nova.compute.manager [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Instance network_info: |[{"id": "06938dfa-9e35-4180-b0e3-9eea613de885", "address": "fa:16:3e:2a:1c:0a", "network": {"id": "6518e147-9c01-4e6f-b77b-4721f647fccf", "bridge": "br-int", "label": "tempest-ServersTestJSON-48688194-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d788eadbca324d08bc7afcac518d6585", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "310b8ba9-edca-4135-863e-f4a786dd4a77", "external-id": "nsx-vlan-transportzone-768", "segmentation_id": 768, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap06938dfa-9e", "ovs_interfaceid": "06938dfa-9e35-4180-b0e3-9eea613de885", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 751.660982] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:2a:1c:0a', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '310b8ba9-edca-4135-863e-f4a786dd4a77', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '06938dfa-9e35-4180-b0e3-9eea613de885', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 751.669034] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Creating folder: Project (d788eadbca324d08bc7afcac518d6585). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 751.669495] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-b4174dca-6370-4f7f-9551-af883db75b0b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 751.678608] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Created folder: Project (d788eadbca324d08bc7afcac518d6585) in parent group-v639722. [ 751.678796] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Creating folder: Instances. Parent ref: group-v639766. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 751.679043] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-137fda7e-3fbe-478b-b0e3-86a1872db731 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 751.686910] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Created folder: Instances in parent group-v639766. [ 751.687040] env[67752]: DEBUG oslo.service.loopingcall [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 751.687224] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 751.687452] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-e8c8702b-23c4-415d-b6a4-4d8a8ebd68d7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 751.708431] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 751.708431] env[67752]: value = "task-3199663" [ 751.708431] env[67752]: _type = "Task" [ 751.708431] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 751.714692] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199663, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 751.786665] env[67752]: DEBUG nova.compute.manager [req-d294e6ff-699e-401c-9146-ebbbc2403ccd req-d70fcda4-2449-4095-a412-342dc35e19e7 service nova] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Received event network-vif-plugged-06938dfa-9e35-4180-b0e3-9eea613de885 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 751.786952] env[67752]: DEBUG oslo_concurrency.lockutils [req-d294e6ff-699e-401c-9146-ebbbc2403ccd req-d70fcda4-2449-4095-a412-342dc35e19e7 service nova] Acquiring lock "02942b29-2343-441e-9e2b-aca2af57c849-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 751.787126] env[67752]: DEBUG oslo_concurrency.lockutils [req-d294e6ff-699e-401c-9146-ebbbc2403ccd req-d70fcda4-2449-4095-a412-342dc35e19e7 service nova] Lock "02942b29-2343-441e-9e2b-aca2af57c849-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 751.787319] env[67752]: DEBUG oslo_concurrency.lockutils [req-d294e6ff-699e-401c-9146-ebbbc2403ccd req-d70fcda4-2449-4095-a412-342dc35e19e7 service nova] Lock "02942b29-2343-441e-9e2b-aca2af57c849-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 751.787538] env[67752]: DEBUG nova.compute.manager [req-d294e6ff-699e-401c-9146-ebbbc2403ccd req-d70fcda4-2449-4095-a412-342dc35e19e7 service nova] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] No waiting events found dispatching network-vif-plugged-06938dfa-9e35-4180-b0e3-9eea613de885 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 751.787685] env[67752]: WARNING nova.compute.manager [req-d294e6ff-699e-401c-9146-ebbbc2403ccd req-d70fcda4-2449-4095-a412-342dc35e19e7 service nova] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Received unexpected event network-vif-plugged-06938dfa-9e35-4180-b0e3-9eea613de885 for instance with vm_state building and task_state spawning. [ 752.216885] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199663, 'name': CreateVM_Task, 'duration_secs': 0.306945} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 752.217133] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 752.217876] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 752.218060] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 752.218388] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 752.218633] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-310df8f4-edf5-4e1c-a08c-e9f860529250 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 752.223519] env[67752]: DEBUG oslo_vmware.api [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Waiting for the task: (returnval){ [ 752.223519] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52037488-889d-e852-be9a-509da8f8f4e3" [ 752.223519] env[67752]: _type = "Task" [ 752.223519] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 752.231372] env[67752]: DEBUG oslo_vmware.api [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52037488-889d-e852-be9a-509da8f8f4e3, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 752.733691] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 752.733960] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 752.734181] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 753.879330] env[67752]: DEBUG nova.compute.manager [req-653e7289-eada-4e0a-a9f1-074d9c33964e req-a41c4f7d-1e30-43be-b47b-6385d966a22e service nova] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Received event network-changed-06938dfa-9e35-4180-b0e3-9eea613de885 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 753.879569] env[67752]: DEBUG nova.compute.manager [req-653e7289-eada-4e0a-a9f1-074d9c33964e req-a41c4f7d-1e30-43be-b47b-6385d966a22e service nova] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Refreshing instance network info cache due to event network-changed-06938dfa-9e35-4180-b0e3-9eea613de885. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 753.879747] env[67752]: DEBUG oslo_concurrency.lockutils [req-653e7289-eada-4e0a-a9f1-074d9c33964e req-a41c4f7d-1e30-43be-b47b-6385d966a22e service nova] Acquiring lock "refresh_cache-02942b29-2343-441e-9e2b-aca2af57c849" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 753.879892] env[67752]: DEBUG oslo_concurrency.lockutils [req-653e7289-eada-4e0a-a9f1-074d9c33964e req-a41c4f7d-1e30-43be-b47b-6385d966a22e service nova] Acquired lock "refresh_cache-02942b29-2343-441e-9e2b-aca2af57c849" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 753.880276] env[67752]: DEBUG nova.network.neutron [req-653e7289-eada-4e0a-a9f1-074d9c33964e req-a41c4f7d-1e30-43be-b47b-6385d966a22e service nova] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Refreshing network info cache for port 06938dfa-9e35-4180-b0e3-9eea613de885 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 754.359876] env[67752]: DEBUG nova.network.neutron [req-653e7289-eada-4e0a-a9f1-074d9c33964e req-a41c4f7d-1e30-43be-b47b-6385d966a22e service nova] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Updated VIF entry in instance network info cache for port 06938dfa-9e35-4180-b0e3-9eea613de885. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 754.360275] env[67752]: DEBUG nova.network.neutron [req-653e7289-eada-4e0a-a9f1-074d9c33964e req-a41c4f7d-1e30-43be-b47b-6385d966a22e service nova] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Updating instance_info_cache with network_info: [{"id": "06938dfa-9e35-4180-b0e3-9eea613de885", "address": "fa:16:3e:2a:1c:0a", "network": {"id": "6518e147-9c01-4e6f-b77b-4721f647fccf", "bridge": "br-int", "label": "tempest-ServersTestJSON-48688194-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d788eadbca324d08bc7afcac518d6585", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "310b8ba9-edca-4135-863e-f4a786dd4a77", "external-id": "nsx-vlan-transportzone-768", "segmentation_id": 768, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap06938dfa-9e", "ovs_interfaceid": "06938dfa-9e35-4180-b0e3-9eea613de885", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 754.371260] env[67752]: DEBUG oslo_concurrency.lockutils [req-653e7289-eada-4e0a-a9f1-074d9c33964e req-a41c4f7d-1e30-43be-b47b-6385d966a22e service nova] Releasing lock "refresh_cache-02942b29-2343-441e-9e2b-aca2af57c849" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 754.371576] env[67752]: DEBUG nova.compute.manager [req-653e7289-eada-4e0a-a9f1-074d9c33964e req-a41c4f7d-1e30-43be-b47b-6385d966a22e service nova] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Received event network-changed-6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 754.371713] env[67752]: DEBUG nova.compute.manager [req-653e7289-eada-4e0a-a9f1-074d9c33964e req-a41c4f7d-1e30-43be-b47b-6385d966a22e service nova] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Refreshing instance network info cache due to event network-changed-6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 754.371918] env[67752]: DEBUG oslo_concurrency.lockutils [req-653e7289-eada-4e0a-a9f1-074d9c33964e req-a41c4f7d-1e30-43be-b47b-6385d966a22e service nova] Acquiring lock "refresh_cache-cbe3fe72-5a91-4c81-ac62-be6592aa806a" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 754.372308] env[67752]: DEBUG oslo_concurrency.lockutils [req-653e7289-eada-4e0a-a9f1-074d9c33964e req-a41c4f7d-1e30-43be-b47b-6385d966a22e service nova] Acquired lock "refresh_cache-cbe3fe72-5a91-4c81-ac62-be6592aa806a" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 754.372308] env[67752]: DEBUG nova.network.neutron [req-653e7289-eada-4e0a-a9f1-074d9c33964e req-a41c4f7d-1e30-43be-b47b-6385d966a22e service nova] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Refreshing network info cache for port 6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 754.889340] env[67752]: DEBUG nova.network.neutron [req-653e7289-eada-4e0a-a9f1-074d9c33964e req-a41c4f7d-1e30-43be-b47b-6385d966a22e service nova] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Updated VIF entry in instance network info cache for port 6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 754.889709] env[67752]: DEBUG nova.network.neutron [req-653e7289-eada-4e0a-a9f1-074d9c33964e req-a41c4f7d-1e30-43be-b47b-6385d966a22e service nova] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Updating instance_info_cache with network_info: [{"id": "6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2", "address": "fa:16:3e:97:43:ec", "network": {"id": "c8fa9bf0-50df-4d49-9e40-03d925f235f2", "bridge": "br-int", "label": "tempest-ServerActionsV293TestJSON-2009741725-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": [{"address": "10.180.180.150", "type": "floating", "version": 4, "meta": {}}]}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "c1a16a3476764b939f260917e4b7391a", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a4fc0575-c8e3-4f3c-b2e1-e10ac2d0cc1a", "external-id": "nsx-vlan-transportzone-256", "segmentation_id": 256, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap6fdef7ca-a3", "ovs_interfaceid": "6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 754.903922] env[67752]: DEBUG oslo_concurrency.lockutils [req-653e7289-eada-4e0a-a9f1-074d9c33964e req-a41c4f7d-1e30-43be-b47b-6385d966a22e service nova] Releasing lock "refresh_cache-cbe3fe72-5a91-4c81-ac62-be6592aa806a" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 766.802536] env[67752]: INFO nova.compute.manager [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Rebuilding instance [ 766.844150] env[67752]: DEBUG nova.compute.manager [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Checking state {{(pid=67752) _get_power_state /opt/stack/nova/nova/compute/manager.py:1782}} [ 766.844891] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5341de12-1d5f-4f35-b512-251318898a2c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.900473] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Powering off the VM {{(pid=67752) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 766.900992] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-4f02ad2a-60ac-443a-b03d-bedf82e16746 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 766.908187] env[67752]: DEBUG oslo_vmware.api [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Waiting for the task: (returnval){ [ 766.908187] env[67752]: value = "task-3199664" [ 766.908187] env[67752]: _type = "Task" [ 766.908187] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 766.915894] env[67752]: DEBUG oslo_vmware.api [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199664, 'name': PowerOffVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 767.423206] env[67752]: DEBUG oslo_vmware.api [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199664, 'name': PowerOffVM_Task, 'duration_secs': 0.196133} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 767.423609] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Powered off the VM {{(pid=67752) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 767.424346] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Powering off the VM {{(pid=67752) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 767.424610] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-13cad54b-81e4-40ac-9bbb-84ad1556a0ee {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.430926] env[67752]: DEBUG oslo_vmware.api [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Waiting for the task: (returnval){ [ 767.430926] env[67752]: value = "task-3199665" [ 767.430926] env[67752]: _type = "Task" [ 767.430926] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 767.439432] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] VM already powered off {{(pid=67752) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1509}} [ 767.439642] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Volume detach. Driver type: vmdk {{(pid=67752) detach_volume /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:646}} [ 767.439913] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] _detach_volume_vmdk: {'driver_volume_type': 'vmdk', 'data': {'volume': 'vm-639741', 'volume_id': '36021ac9-93ea-424f-9037-401e63e60004', 'name': 'volume-36021ac9-93ea-424f-9037-401e63e60004', 'profile_id': None, 'qos_specs': None, 'access_mode': 'rw', 'encrypted': False, 'cacheable': False}, 'status': 'reserved', 'instance': 'cbe3fe72-5a91-4c81-ac62-be6592aa806a', 'attached_at': '', 'detached_at': '', 'volume_id': '36021ac9-93ea-424f-9037-401e63e60004', 'serial': '36021ac9-93ea-424f-9037-401e63e60004'} {{(pid=67752) _detach_volume_vmdk /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:571}} [ 767.440663] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-43f0875c-bbfd-43e2-9e6f-3ec77a44dc1f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.459045] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-39b9aa96-76d9-4b1d-b8ad-ea70d7d5a65d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.465101] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3042023-4fd5-4222-9d51-fc0ee5025777 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.483332] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-33f9ff52-55c4-410a-b33e-4e6f1c6552f4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.498424] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] The volume has not been displaced from its original location: [datastore2] volume-36021ac9-93ea-424f-9037-401e63e60004/volume-36021ac9-93ea-424f-9037-401e63e60004.vmdk. No consolidation needed. {{(pid=67752) _consolidate_vmdk_volume /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:504}} [ 767.503515] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Reconfiguring VM instance instance-0000000e to detach disk 2000 {{(pid=67752) detach_disk_from_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:122}} [ 767.503771] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-49b4f4f1-2dfe-4496-be0a-8b69dbc16513 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 767.522035] env[67752]: DEBUG oslo_vmware.api [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Waiting for the task: (returnval){ [ 767.522035] env[67752]: value = "task-3199666" [ 767.522035] env[67752]: _type = "Task" [ 767.522035] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 767.529720] env[67752]: DEBUG oslo_vmware.api [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199666, 'name': ReconfigVM_Task} progress is 5%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 768.032178] env[67752]: DEBUG oslo_vmware.api [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199666, 'name': ReconfigVM_Task, 'duration_secs': 0.149735} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 768.032543] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Reconfigured VM instance instance-0000000e to detach disk 2000 {{(pid=67752) detach_disk_from_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:127}} [ 768.037115] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-ddb79555-60f2-470b-8613-694c08244ae1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.051255] env[67752]: DEBUG oslo_vmware.api [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Waiting for the task: (returnval){ [ 768.051255] env[67752]: value = "task-3199667" [ 768.051255] env[67752]: _type = "Task" [ 768.051255] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 768.058659] env[67752]: DEBUG oslo_vmware.api [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199667, 'name': ReconfigVM_Task} progress is 5%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 768.561219] env[67752]: DEBUG oslo_vmware.api [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199667, 'name': ReconfigVM_Task, 'duration_secs': 0.118554} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 768.561528] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Detached VMDK: {'driver_volume_type': 'vmdk', 'data': {'volume': 'vm-639741', 'volume_id': '36021ac9-93ea-424f-9037-401e63e60004', 'name': 'volume-36021ac9-93ea-424f-9037-401e63e60004', 'profile_id': None, 'qos_specs': None, 'access_mode': 'rw', 'encrypted': False, 'cacheable': False}, 'status': 'reserved', 'instance': 'cbe3fe72-5a91-4c81-ac62-be6592aa806a', 'attached_at': '', 'detached_at': '', 'volume_id': '36021ac9-93ea-424f-9037-401e63e60004', 'serial': '36021ac9-93ea-424f-9037-401e63e60004'} {{(pid=67752) _detach_volume_vmdk /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:605}} [ 768.561798] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 768.562775] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7b273345-ac22-45ca-95c4-aa899a0244c3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.570446] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 768.570446] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-5cfad751-6731-43a0-b6c3-6aa8cf203fd9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.625310] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 768.625499] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 768.625597] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Deleting the datastore file [datastore2] cbe3fe72-5a91-4c81-ac62-be6592aa806a {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 768.625849] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-6b7941d3-26bb-4118-9e65-55e2c388bb27 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 768.631620] env[67752]: DEBUG oslo_vmware.api [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Waiting for the task: (returnval){ [ 768.631620] env[67752]: value = "task-3199669" [ 768.631620] env[67752]: _type = "Task" [ 768.631620] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 768.639042] env[67752]: DEBUG oslo_vmware.api [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199669, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 769.141408] env[67752]: DEBUG oslo_vmware.api [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Task: {'id': task-3199669, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.078496} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 769.141683] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 769.141890] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 769.142086] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 769.202929] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Volume detach. Driver type: vmdk {{(pid=67752) detach_volume /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:646}} [ 769.203344] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-279ac789-f333-4f29-9db6-df6c8a4b233e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 769.212478] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fb47e43e-c2a7-4a3b-9365-c94ab2cc0acb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 769.242260] env[67752]: ERROR nova.compute.manager [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Failed to detach volume 36021ac9-93ea-424f-9037-401e63e60004 from /dev/sda: nova.exception.InstanceNotFound: Instance cbe3fe72-5a91-4c81-ac62-be6592aa806a could not be found. [ 769.242260] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Traceback (most recent call last): [ 769.242260] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/compute/manager.py", line 4133, in _do_rebuild_instance [ 769.242260] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] self.driver.rebuild(**kwargs) [ 769.242260] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/virt/driver.py", line 390, in rebuild [ 769.242260] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] raise NotImplementedError() [ 769.242260] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] NotImplementedError [ 769.242260] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] [ 769.242260] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] During handling of the above exception, another exception occurred: [ 769.242260] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] [ 769.242260] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Traceback (most recent call last): [ 769.242260] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/compute/manager.py", line 3556, in _detach_root_volume [ 769.242260] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] self.driver.detach_volume(context, old_connection_info, [ 769.242745] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 552, in detach_volume [ 769.242745] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] return self._volumeops.detach_volume(connection_info, instance) [ 769.242745] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/virt/vmwareapi/volumeops.py", line 649, in detach_volume [ 769.242745] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] self._detach_volume_vmdk(connection_info, instance) [ 769.242745] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/virt/vmwareapi/volumeops.py", line 569, in _detach_volume_vmdk [ 769.242745] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] vm_ref = vm_util.get_vm_ref(self._session, instance) [ 769.242745] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1135, in get_vm_ref [ 769.242745] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] stable_ref.fetch_moref(session) [ 769.242745] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1126, in fetch_moref [ 769.242745] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] raise exception.InstanceNotFound(instance_id=self._uuid) [ 769.242745] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] nova.exception.InstanceNotFound: Instance cbe3fe72-5a91-4c81-ac62-be6592aa806a could not be found. [ 769.242745] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] [ 769.383471] env[67752]: DEBUG nova.compute.utils [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Build of instance cbe3fe72-5a91-4c81-ac62-be6592aa806a aborted: Failed to rebuild volume backed instance. {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 769.386285] env[67752]: ERROR nova.compute.manager [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Setting instance vm_state to ERROR: nova.exception.BuildAbortException: Build of instance cbe3fe72-5a91-4c81-ac62-be6592aa806a aborted: Failed to rebuild volume backed instance. [ 769.386285] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Traceback (most recent call last): [ 769.386285] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/compute/manager.py", line 4133, in _do_rebuild_instance [ 769.386285] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] self.driver.rebuild(**kwargs) [ 769.386285] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/virt/driver.py", line 390, in rebuild [ 769.386285] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] raise NotImplementedError() [ 769.386285] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] NotImplementedError [ 769.386285] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] [ 769.386285] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] During handling of the above exception, another exception occurred: [ 769.386285] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] [ 769.386285] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Traceback (most recent call last): [ 769.386285] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/compute/manager.py", line 3591, in _rebuild_volume_backed_instance [ 769.386892] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] self._detach_root_volume(context, instance, root_bdm) [ 769.386892] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/compute/manager.py", line 3570, in _detach_root_volume [ 769.386892] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] with excutils.save_and_reraise_exception(): [ 769.386892] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 769.386892] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] self.force_reraise() [ 769.386892] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 769.386892] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] raise self.value [ 769.386892] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/compute/manager.py", line 3556, in _detach_root_volume [ 769.386892] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] self.driver.detach_volume(context, old_connection_info, [ 769.386892] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 552, in detach_volume [ 769.386892] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] return self._volumeops.detach_volume(connection_info, instance) [ 769.386892] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/virt/vmwareapi/volumeops.py", line 649, in detach_volume [ 769.386892] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] self._detach_volume_vmdk(connection_info, instance) [ 769.387209] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/virt/vmwareapi/volumeops.py", line 569, in _detach_volume_vmdk [ 769.387209] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] vm_ref = vm_util.get_vm_ref(self._session, instance) [ 769.387209] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1135, in get_vm_ref [ 769.387209] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] stable_ref.fetch_moref(session) [ 769.387209] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1126, in fetch_moref [ 769.387209] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] raise exception.InstanceNotFound(instance_id=self._uuid) [ 769.387209] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] nova.exception.InstanceNotFound: Instance cbe3fe72-5a91-4c81-ac62-be6592aa806a could not be found. [ 769.387209] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] [ 769.387209] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] During handling of the above exception, another exception occurred: [ 769.387209] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] [ 769.387209] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Traceback (most recent call last): [ 769.387209] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/compute/manager.py", line 10856, in _error_out_instance_on_exception [ 769.387209] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] yield [ 769.387209] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/compute/manager.py", line 3859, in rebuild_instance [ 769.387688] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] self._do_rebuild_instance_with_claim( [ 769.387688] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/compute/manager.py", line 3945, in _do_rebuild_instance_with_claim [ 769.387688] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] self._do_rebuild_instance( [ 769.387688] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/compute/manager.py", line 4137, in _do_rebuild_instance [ 769.387688] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] self._rebuild_default_impl(**kwargs) [ 769.387688] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/compute/manager.py", line 3714, in _rebuild_default_impl [ 769.387688] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] self._rebuild_volume_backed_instance( [ 769.387688] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] File "/opt/stack/nova/nova/compute/manager.py", line 3606, in _rebuild_volume_backed_instance [ 769.387688] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] raise exception.BuildAbortException( [ 769.387688] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] nova.exception.BuildAbortException: Build of instance cbe3fe72-5a91-4c81-ac62-be6592aa806a aborted: Failed to rebuild volume backed instance. [ 769.387688] env[67752]: ERROR nova.compute.manager [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] [ 769.498565] env[67752]: DEBUG oslo_concurrency.lockutils [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 769.498824] env[67752]: DEBUG oslo_concurrency.lockutils [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 769.888501] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4f0641fb-2754-430c-9aac-279efec88c95 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 769.896345] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93bd0ef9-3dee-4284-a9d1-d47ee5c8816c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 769.926169] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-515773f2-6a61-43f4-8af5-27aeea616457 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 769.933249] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94e7abe2-b621-48b7-9b3b-3b85b99f54c5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 769.946339] env[67752]: DEBUG nova.compute.provider_tree [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 769.972954] env[67752]: ERROR nova.scheduler.client.report [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [req-9db994cc-ca2c-42f9-8619-3e0eb0acbd3b] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID c7141b98-bb6c-46bd-b37f-a8e913940199. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-9db994cc-ca2c-42f9-8619-3e0eb0acbd3b"}]}: nova.exception.BuildAbortException: Build of instance cbe3fe72-5a91-4c81-ac62-be6592aa806a aborted: Failed to rebuild volume backed instance. [ 769.988728] env[67752]: DEBUG nova.scheduler.client.report [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Refreshing inventories for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:804}} [ 770.002739] env[67752]: DEBUG nova.scheduler.client.report [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Updating ProviderTree inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:768}} [ 770.002970] env[67752]: DEBUG nova.compute.provider_tree [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 770.014445] env[67752]: DEBUG nova.scheduler.client.report [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Refreshing aggregate associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, aggregates: None {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:813}} [ 770.033726] env[67752]: DEBUG nova.scheduler.client.report [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Refreshing trait associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, traits: COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:825}} [ 770.301611] env[67752]: DEBUG oslo_concurrency.lockutils [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Acquiring lock "cbe3fe72-5a91-4c81-ac62-be6592aa806a" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 770.301872] env[67752]: DEBUG oslo_concurrency.lockutils [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Lock "cbe3fe72-5a91-4c81-ac62-be6592aa806a" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 770.302061] env[67752]: DEBUG oslo_concurrency.lockutils [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Acquiring lock "cbe3fe72-5a91-4c81-ac62-be6592aa806a-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 770.302234] env[67752]: DEBUG oslo_concurrency.lockutils [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Lock "cbe3fe72-5a91-4c81-ac62-be6592aa806a-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 770.302419] env[67752]: DEBUG oslo_concurrency.lockutils [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Lock "cbe3fe72-5a91-4c81-ac62-be6592aa806a-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 770.308024] env[67752]: INFO nova.compute.manager [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Terminating instance [ 770.310073] env[67752]: DEBUG nova.compute.manager [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 770.310738] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-da868473-cf61-41f2-afd6-c544344809a3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 770.319250] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0603c054-fb98-4e33-816d-fc6ecf1ea50c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 770.351065] env[67752]: WARNING nova.virt.vmwareapi.driver [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Instance does not exists. Proceeding to delete instance properties on datastore: nova.exception.InstanceNotFound: Instance cbe3fe72-5a91-4c81-ac62-be6592aa806a could not be found. [ 770.351277] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 770.353745] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-9c6dd8ee-9cba-46d8-a18e-653af1fca21b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 770.361044] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bbde55cd-9062-4f74-9b7c-9645fc3fa932 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 770.392174] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance cbe3fe72-5a91-4c81-ac62-be6592aa806a could not be found. [ 770.392461] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 770.392672] env[67752]: INFO nova.compute.manager [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Took 0.08 seconds to destroy the instance on the hypervisor. [ 770.392949] env[67752]: DEBUG oslo.service.loopingcall [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 770.395609] env[67752]: DEBUG nova.compute.manager [-] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 770.395735] env[67752]: DEBUG nova.network.neutron [-] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 770.460929] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c78e62c-1272-432d-8789-5c135b2e1439 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 770.468565] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db65b24f-f812-4b3a-9256-220f3536a888 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 770.499039] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cfb149ca-7e6a-4f89-a723-d2a0b7063004 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 770.506206] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a613df5-6320-43ff-a94e-491cf6882d7c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 770.518831] env[67752]: DEBUG nova.compute.provider_tree [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 770.556216] env[67752]: DEBUG nova.scheduler.client.report [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Updated inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with generation 46 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:957}} [ 770.556484] env[67752]: DEBUG nova.compute.provider_tree [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Updating resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 generation from 46 to 47 during operation: update_inventory {{(pid=67752) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 770.556669] env[67752]: DEBUG nova.compute.provider_tree [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 770.578764] env[67752]: DEBUG oslo_concurrency.lockutils [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 1.080s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 770.578999] env[67752]: INFO nova.compute.manager [None req-54c1ea43-e72c-4de5-9485-0595e74a2a30 tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Successfully reverted task state from rebuilding on failure for instance. [ 771.436806] env[67752]: DEBUG nova.network.neutron [-] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 771.456627] env[67752]: INFO nova.compute.manager [-] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Took 1.06 seconds to deallocate network for instance. [ 771.547773] env[67752]: INFO nova.compute.manager [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Took 0.09 seconds to detach 1 volumes for instance. [ 771.550992] env[67752]: DEBUG nova.compute.manager [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Deleting volume: 36021ac9-93ea-424f-9037-401e63e60004 {{(pid=67752) _cleanup_volumes /opt/stack/nova/nova/compute/manager.py:3238}} [ 771.630433] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 771.661550] env[67752]: DEBUG oslo_concurrency.lockutils [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 771.661839] env[67752]: DEBUG oslo_concurrency.lockutils [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 771.662129] env[67752]: DEBUG nova.objects.instance [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Lazy-loading 'resources' on Instance uuid cbe3fe72-5a91-4c81-ac62-be6592aa806a {{(pid=67752) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 771.892020] env[67752]: DEBUG nova.compute.manager [req-cc9f7c38-6eeb-48fc-919e-9269780be3b8 req-fd62a02e-4227-4b75-a17e-6b19446e89d8 service nova] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Received event network-vif-deleted-6fdef7ca-a31b-4a5e-9edd-fedddf37b9a2 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 772.151654] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-220c99b4-7409-482c-b47f-349eab0e5179 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.159291] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-335c6128-adaf-468b-b5d3-1282bee0c82d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.189568] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1064d89-fb5b-4dd6-a4fd-df54c92ce1f3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.197270] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-be7b5719-6f61-4905-ba65-b867fa5fcf9a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 772.210260] env[67752]: DEBUG nova.compute.provider_tree [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 772.225597] env[67752]: DEBUG nova.scheduler.client.report [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 772.253109] env[67752]: DEBUG oslo_concurrency.lockutils [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 0.589s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 772.325264] env[67752]: DEBUG oslo_concurrency.lockutils [None req-736513ce-cd4f-4052-a328-6e62b2c12f0f tempest-ServerActionsV293TestJSON-32365216 tempest-ServerActionsV293TestJSON-32365216-project-member] Lock "cbe3fe72-5a91-4c81-ac62-be6592aa806a" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.023s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 773.636486] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 773.636486] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 773.636486] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 773.661522] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 773.661725] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 773.661931] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 773.662111] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 773.662293] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 773.662446] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 773.662655] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 773.662803] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 773.662942] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 773.663109] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 773.663341] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 773.664036] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 774.659417] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 775.634609] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 775.634864] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 775.635038] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 775.635189] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 775.635342] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 775.648407] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 775.648644] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 775.648826] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 775.648983] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 775.650128] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-86771198-3529-4cc6-9afb-13273946ff23 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 775.658799] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-121dd1ed-7598-406d-a931-8c06441b1e20 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 775.672463] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d51ee192-7b8e-411f-964b-bf4ebf27d00d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 775.678718] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f8d2629-f7fb-4b75-b226-7c42af68704d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 775.708283] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180934MB free_disk=125GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 775.708426] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 775.708626] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 775.799391] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 775.799556] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fc4da5fa-1921-44e3-9100-042391d4ccb9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 775.799688] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 39479042-b891-4f25-9992-47312c6b6b43 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 775.799809] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 775.799928] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 775.800061] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b0cc1acd-0e33-4547-a43b-549b3de81728 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 775.800182] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance eca7c94f-9bd1-4c15-988e-d4e33086ba40 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 775.800299] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ff4895f8-0374-4b34-a5e8-d3ba24f840a5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 775.800421] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ca18af6b-6b80-44f3-bd8b-f61e2596acd7 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 775.800563] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 02942b29-2343-441e-9e2b-aca2af57c849 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 775.812244] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ebd6c719-32ae-410c-a444-db7ad3f8a9c5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 775.830191] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f528dc15-1a86-4af5-ac45-536fe3dfe101 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 775.841657] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 4c0873a7-b8e5-4d3e-bdc0-3d236e3b9df6 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 775.854131] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e3c2e1cf-cae5-474f-9f61-96d79815d846 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 775.863931] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f22bcbae-0dcb-4666-b5e7-00c560e016c3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 775.874132] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c38268b5-b35d-4d8d-aa03-72100e2889f7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 775.884610] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c6a7f89e-d842-4b34-af36-ca60f12bcddd has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 775.895381] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fa9cdf6d-032f-4069-a78e-fb16b4d2c457 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 775.908970] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 775.919046] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 40aaf487-275a-4e75-ac6a-127371e73b2c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 775.929121] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d2104921-5ec1-4fb8-8034-52dcdc403a92 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 775.941659] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 16f0124d-146a-442a-8738-65f562b928e8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 775.971426] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 8503063e-2888-44c6-bc2a-e9c82be791e2 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 775.985462] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 631002d2-c268-4a6a-9945-c0898b886b93 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 775.997714] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 67812e2e-8827-4368-9790-a909c5ebb56b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 776.013079] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 5ed08031-0ef8-4766-951a-37b315c55dc7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 776.022751] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 436d243a-d6d9-4958-94a8-7b09f5d9fdf3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 776.037900] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f8b03251-6f23-4533-a917-f3fdac87888b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 776.050613] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 58d3b02f-3aac-4beb-a111-44f50c7f1bc8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 776.064190] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 53eb8b0c-aa50-4d7f-8027-2725c815f134 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 776.078183] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 411bafb7-ef72-4529-b3f3-0b4955f23788 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 776.088125] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ada70883-9bef-496e-a65f-ca84f6ee6211 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 776.089208] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 776.089529] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 776.501985] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b60a492e-796a-4d3a-91ed-597bde005a19 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 776.510280] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0e1bb6c-0a17-4b81-8d84-6a3bd068ea23 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 776.543506] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0959d817-cd46-413b-a26f-add00d62aa32 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 776.551397] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d2310a55-038f-4ffc-910d-6f59e7c465fd {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 776.565480] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 776.577189] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 776.593189] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 776.593189] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.884s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 777.593996] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 777.634869] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 779.179291] env[67752]: DEBUG oslo_concurrency.lockutils [None req-cb3b4342-144b-4ab6-9200-e49e2d28607f tempest-ServerRescueTestJSONUnderV235-1587357604 tempest-ServerRescueTestJSONUnderV235-1587357604-project-member] Acquiring lock "62f05780-7639-4a4f-975f-3f0863118c99" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 779.179573] env[67752]: DEBUG oslo_concurrency.lockutils [None req-cb3b4342-144b-4ab6-9200-e49e2d28607f tempest-ServerRescueTestJSONUnderV235-1587357604 tempest-ServerRescueTestJSONUnderV235-1587357604-project-member] Lock "62f05780-7639-4a4f-975f-3f0863118c99" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 780.861172] env[67752]: DEBUG oslo_concurrency.lockutils [None req-417c37a5-5d3b-4858-bbac-40938f3a45e2 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Acquiring lock "b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 783.496037] env[67752]: WARNING oslo_vmware.rw_handles [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 783.496037] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 783.496037] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 783.496037] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 783.496037] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 783.496037] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 783.496037] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 783.496037] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 783.496037] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 783.496037] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 783.496037] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 783.496037] env[67752]: ERROR oslo_vmware.rw_handles [ 783.497178] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/2ceff526-cc04-486a-928b-7d151165f80c/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 783.498514] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 783.498763] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Copying Virtual Disk [datastore2] vmware_temp/2ceff526-cc04-486a-928b-7d151165f80c/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/2ceff526-cc04-486a-928b-7d151165f80c/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 783.499118] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-139112fe-dee4-41a7-9d36-86686be8c957 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 783.507269] env[67752]: DEBUG oslo_vmware.api [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Waiting for the task: (returnval){ [ 783.507269] env[67752]: value = "task-3199671" [ 783.507269] env[67752]: _type = "Task" [ 783.507269] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 783.515349] env[67752]: DEBUG oslo_vmware.api [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Task: {'id': task-3199671, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 784.018492] env[67752]: DEBUG oslo_vmware.exceptions [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 784.018824] env[67752]: DEBUG oslo_concurrency.lockutils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 784.019429] env[67752]: ERROR nova.compute.manager [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 784.019429] env[67752]: Faults: ['InvalidArgument'] [ 784.019429] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Traceback (most recent call last): [ 784.019429] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 784.019429] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] yield resources [ 784.019429] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 784.019429] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] self.driver.spawn(context, instance, image_meta, [ 784.019429] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 784.019429] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 784.019429] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 784.019429] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] self._fetch_image_if_missing(context, vi) [ 784.019429] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 784.019764] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] image_cache(vi, tmp_image_ds_loc) [ 784.019764] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 784.019764] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] vm_util.copy_virtual_disk( [ 784.019764] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 784.019764] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] session._wait_for_task(vmdk_copy_task) [ 784.019764] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 784.019764] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] return self.wait_for_task(task_ref) [ 784.019764] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 784.019764] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] return evt.wait() [ 784.019764] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 784.019764] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] result = hub.switch() [ 784.019764] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 784.019764] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] return self.greenlet.switch() [ 784.020203] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 784.020203] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] self.f(*self.args, **self.kw) [ 784.020203] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 784.020203] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] raise exceptions.translate_fault(task_info.error) [ 784.020203] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 784.020203] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Faults: ['InvalidArgument'] [ 784.020203] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] [ 784.020203] env[67752]: INFO nova.compute.manager [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Terminating instance [ 784.021384] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 784.021599] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 784.022234] env[67752]: DEBUG nova.compute.manager [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 784.022426] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 784.022650] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-84f48a7e-0ba5-4a5f-805a-a15d311dc2fb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 784.024936] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d5b3826-20ed-41c3-bb25-e7850e63996d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 784.031633] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 784.031843] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-748df8ab-b5d2-4ad5-88f2-e2a3e93112a7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 784.034020] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 784.034198] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 784.035168] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-7e24205f-26b2-419a-91d5-f06ba294199c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 784.039547] env[67752]: DEBUG oslo_vmware.api [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Waiting for the task: (returnval){ [ 784.039547] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52b9a5ff-823d-042b-6e76-495ef192c026" [ 784.039547] env[67752]: _type = "Task" [ 784.039547] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 784.046752] env[67752]: DEBUG oslo_vmware.api [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52b9a5ff-823d-042b-6e76-495ef192c026, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 784.102595] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 784.102893] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 784.103098] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Deleting the datastore file [datastore2] b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 784.103367] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-9c77c3a0-314e-4f44-9af7-691864de35af {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 784.111390] env[67752]: DEBUG oslo_vmware.api [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Waiting for the task: (returnval){ [ 784.111390] env[67752]: value = "task-3199673" [ 784.111390] env[67752]: _type = "Task" [ 784.111390] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 784.117604] env[67752]: DEBUG oslo_vmware.api [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Task: {'id': task-3199673, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 784.549813] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 784.550146] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Creating directory with path [datastore2] vmware_temp/b9454410-fb34-45cb-b3b4-032dadc5f7b9/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 784.550318] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-3bfa1a7e-974f-4977-9199-0eaafbcbd429 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 784.561727] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Created directory with path [datastore2] vmware_temp/b9454410-fb34-45cb-b3b4-032dadc5f7b9/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 784.561910] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Fetch image to [datastore2] vmware_temp/b9454410-fb34-45cb-b3b4-032dadc5f7b9/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 784.562098] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/b9454410-fb34-45cb-b3b4-032dadc5f7b9/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 784.562841] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3e6cc07-0a4e-4ec4-b141-562befa5bedc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 784.570484] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-452eaa10-aff3-43f0-8d3b-b02384ebf6d5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 784.579300] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9a26d179-870a-42d7-8c66-54edd0273312 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 784.609926] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9528be7b-ab9a-4ca6-8edb-1d47b2c71c16 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 784.620272] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-d05886ae-8638-4cf8-bef1-ce4c6dfd5419 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 784.621857] env[67752]: DEBUG oslo_vmware.api [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Task: {'id': task-3199673, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.066623} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 784.622107] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 784.622290] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 784.622455] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 784.622631] env[67752]: INFO nova.compute.manager [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Took 0.60 seconds to destroy the instance on the hypervisor. [ 784.624894] env[67752]: DEBUG nova.compute.claims [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 784.625139] env[67752]: DEBUG oslo_concurrency.lockutils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 784.625369] env[67752]: DEBUG oslo_concurrency.lockutils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 784.644498] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 784.695716] env[67752]: DEBUG oslo_vmware.rw_handles [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/b9454410-fb34-45cb-b3b4-032dadc5f7b9/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 784.758275] env[67752]: DEBUG oslo_vmware.rw_handles [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 784.758481] env[67752]: DEBUG oslo_vmware.rw_handles [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/b9454410-fb34-45cb-b3b4-032dadc5f7b9/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 785.108235] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-08afedb1-7635-4f94-bb1b-815b626cfaf4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 785.115648] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d0d84caf-8f99-40bd-8322-369ba26fe857 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 785.145099] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e038b435-c6a3-46ed-8b20-d4a4a557c81c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 785.152080] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7fc73d33-55b2-4ddf-90e6-8544cdd7f4f0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 785.165063] env[67752]: DEBUG nova.compute.provider_tree [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 785.199201] env[67752]: ERROR nova.scheduler.client.report [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [req-21f96c08-95ff-41b9-a5b4-a2a21c5a31c5] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID c7141b98-bb6c-46bd-b37f-a8e913940199. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-21f96c08-95ff-41b9-a5b4-a2a21c5a31c5"}]}: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 785.230329] env[67752]: DEBUG nova.scheduler.client.report [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Refreshing inventories for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:804}} [ 785.244816] env[67752]: DEBUG nova.scheduler.client.report [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Updating ProviderTree inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:768}} [ 785.245187] env[67752]: DEBUG nova.compute.provider_tree [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 785.258893] env[67752]: DEBUG nova.scheduler.client.report [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Refreshing aggregate associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, aggregates: None {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:813}} [ 785.279668] env[67752]: DEBUG nova.scheduler.client.report [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Refreshing trait associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, traits: COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:825}} [ 785.652072] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a26a1494-21ef-4056-97fd-056604d33e9c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 785.659602] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fd73a831-751e-4a29-a4fd-e4c3a178b9d1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 785.689679] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad317122-6853-49d6-a145-c30a3dbdf050 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 785.697269] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8456b231-f136-4463-a7e1-a5c9b2e7b959 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 785.709853] env[67752]: DEBUG nova.compute.provider_tree [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 785.747696] env[67752]: DEBUG nova.scheduler.client.report [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Updated inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with generation 48 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:957}} [ 785.747972] env[67752]: DEBUG nova.compute.provider_tree [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Updating resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 generation from 48 to 49 during operation: update_inventory {{(pid=67752) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 785.748200] env[67752]: DEBUG nova.compute.provider_tree [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 785.765655] env[67752]: DEBUG oslo_concurrency.lockutils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.140s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 785.766224] env[67752]: ERROR nova.compute.manager [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 785.766224] env[67752]: Faults: ['InvalidArgument'] [ 785.766224] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Traceback (most recent call last): [ 785.766224] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 785.766224] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] self.driver.spawn(context, instance, image_meta, [ 785.766224] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 785.766224] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] self._vmops.spawn(context, instance, image_meta, injected_files, [ 785.766224] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 785.766224] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] self._fetch_image_if_missing(context, vi) [ 785.766224] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 785.766224] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] image_cache(vi, tmp_image_ds_loc) [ 785.766224] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 785.766598] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] vm_util.copy_virtual_disk( [ 785.766598] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 785.766598] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] session._wait_for_task(vmdk_copy_task) [ 785.766598] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 785.766598] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] return self.wait_for_task(task_ref) [ 785.766598] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 785.766598] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] return evt.wait() [ 785.766598] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 785.766598] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] result = hub.switch() [ 785.766598] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 785.766598] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] return self.greenlet.switch() [ 785.766598] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 785.766598] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] self.f(*self.args, **self.kw) [ 785.766945] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 785.766945] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] raise exceptions.translate_fault(task_info.error) [ 785.766945] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 785.766945] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Faults: ['InvalidArgument'] [ 785.766945] env[67752]: ERROR nova.compute.manager [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] [ 785.766945] env[67752]: DEBUG nova.compute.utils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 785.768671] env[67752]: DEBUG nova.compute.manager [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Build of instance b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3 was re-scheduled: A specified parameter was not correct: fileType [ 785.768671] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 785.769048] env[67752]: DEBUG nova.compute.manager [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 785.769235] env[67752]: DEBUG nova.compute.manager [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 785.769406] env[67752]: DEBUG nova.compute.manager [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 785.769569] env[67752]: DEBUG nova.network.neutron [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 786.305902] env[67752]: DEBUG nova.network.neutron [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 786.317773] env[67752]: INFO nova.compute.manager [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Took 0.55 seconds to deallocate network for instance. [ 786.427942] env[67752]: INFO nova.scheduler.client.report [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Deleted allocations for instance b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3 [ 786.445775] env[67752]: DEBUG oslo_concurrency.lockutils [None req-eb9a4639-9f58-4eaf-b94f-df8559d7a196 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Lock "b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 204.435s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 786.446962] env[67752]: DEBUG oslo_concurrency.lockutils [None req-417c37a5-5d3b-4858-bbac-40938f3a45e2 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Lock "b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 5.586s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 786.447259] env[67752]: DEBUG oslo_concurrency.lockutils [None req-417c37a5-5d3b-4858-bbac-40938f3a45e2 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Acquiring lock "b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 786.447483] env[67752]: DEBUG oslo_concurrency.lockutils [None req-417c37a5-5d3b-4858-bbac-40938f3a45e2 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Lock "b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 786.447652] env[67752]: DEBUG oslo_concurrency.lockutils [None req-417c37a5-5d3b-4858-bbac-40938f3a45e2 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Lock "b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 786.451758] env[67752]: INFO nova.compute.manager [None req-417c37a5-5d3b-4858-bbac-40938f3a45e2 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Terminating instance [ 786.453792] env[67752]: DEBUG nova.compute.manager [None req-417c37a5-5d3b-4858-bbac-40938f3a45e2 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 786.454032] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-417c37a5-5d3b-4858-bbac-40938f3a45e2 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 786.454497] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-399f3be9-3a03-46f8-a8ab-dd1eac7ab198 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 786.459968] env[67752]: DEBUG nova.compute.manager [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 786.466778] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0a576d3-532e-4d8d-bfc1-0d124bf63e9f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 786.499886] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-417c37a5-5d3b-4858-bbac-40938f3a45e2 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3 could not be found. [ 786.500121] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-417c37a5-5d3b-4858-bbac-40938f3a45e2 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 786.500307] env[67752]: INFO nova.compute.manager [None req-417c37a5-5d3b-4858-bbac-40938f3a45e2 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Took 0.05 seconds to destroy the instance on the hypervisor. [ 786.500547] env[67752]: DEBUG oslo.service.loopingcall [None req-417c37a5-5d3b-4858-bbac-40938f3a45e2 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 786.503596] env[67752]: DEBUG nova.compute.manager [-] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 786.503723] env[67752]: DEBUG nova.network.neutron [-] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 786.518696] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 786.518929] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 786.520390] env[67752]: INFO nova.compute.claims [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 786.541560] env[67752]: DEBUG nova.network.neutron [-] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 786.554826] env[67752]: INFO nova.compute.manager [-] [instance: b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3] Took 0.05 seconds to deallocate network for instance. [ 786.654278] env[67752]: DEBUG oslo_concurrency.lockutils [None req-417c37a5-5d3b-4858-bbac-40938f3a45e2 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802 tempest-FloatingIPsAssociationNegativeTestJSON-1807430802-project-member] Lock "b2493b3f-7ea0-48fe-85d6-e27d47d0c9b3" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.207s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 786.952104] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6242a9a1-8705-4f82-ada6-64c49a0eee67 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 786.959655] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8227ccb6-2d30-4e10-be1c-e6601cb29255 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 786.989656] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-352facef-31e5-4a04-9bad-6517b1d26374 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 786.996742] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-26cfa13d-fe37-4143-b611-ee1fc1e5b158 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 787.009547] env[67752]: DEBUG nova.compute.provider_tree [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 787.018260] env[67752]: DEBUG nova.scheduler.client.report [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 787.043912] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.525s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 787.044474] env[67752]: DEBUG nova.compute.manager [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 787.081942] env[67752]: DEBUG nova.compute.utils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 787.083285] env[67752]: DEBUG nova.compute.manager [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 787.083455] env[67752]: DEBUG nova.network.neutron [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 787.099903] env[67752]: DEBUG nova.compute.manager [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 787.166394] env[67752]: DEBUG nova.compute.manager [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 787.192398] env[67752]: DEBUG nova.virt.hardware [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 787.192641] env[67752]: DEBUG nova.virt.hardware [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 787.192811] env[67752]: DEBUG nova.virt.hardware [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 787.193067] env[67752]: DEBUG nova.virt.hardware [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 787.193186] env[67752]: DEBUG nova.virt.hardware [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 787.193336] env[67752]: DEBUG nova.virt.hardware [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 787.193863] env[67752]: DEBUG nova.virt.hardware [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 787.193863] env[67752]: DEBUG nova.virt.hardware [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 787.193863] env[67752]: DEBUG nova.virt.hardware [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 787.194539] env[67752]: DEBUG nova.virt.hardware [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 787.194539] env[67752]: DEBUG nova.virt.hardware [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 787.195364] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-537380bd-ba3f-4257-ac0b-4a7493d2896c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 787.203652] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-71daec92-135f-4189-b8f2-b33ce183215d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 787.346325] env[67752]: DEBUG nova.policy [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '88759e25c9b54ed99ed28ef49b0f6eb0', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd0bab7da534b4669b94441f5b1330b2f', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 788.088024] env[67752]: DEBUG nova.network.neutron [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Successfully created port: 8f7da60c-1ca7-47ee-b944-302dad820eec {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 789.013713] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5b74ab26-f6dc-41b2-8498-174d0fc093f2 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Acquiring lock "fc4da5fa-1921-44e3-9100-042391d4ccb9" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 789.289609] env[67752]: DEBUG nova.network.neutron [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Successfully updated port: 8f7da60c-1ca7-47ee-b944-302dad820eec {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 789.301823] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Acquiring lock "refresh_cache-ebd6c719-32ae-410c-a444-db7ad3f8a9c5" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 789.301924] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Acquired lock "refresh_cache-ebd6c719-32ae-410c-a444-db7ad3f8a9c5" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 789.302226] env[67752]: DEBUG nova.network.neutron [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 789.607171] env[67752]: DEBUG nova.network.neutron [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 789.838958] env[67752]: DEBUG nova.compute.manager [req-3280b973-52ef-41fa-940b-2e18c376b6ed req-650a9f3f-6d17-4042-90a4-4a7b2da484b2 service nova] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Received event network-vif-plugged-8f7da60c-1ca7-47ee-b944-302dad820eec {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 789.839619] env[67752]: DEBUG oslo_concurrency.lockutils [req-3280b973-52ef-41fa-940b-2e18c376b6ed req-650a9f3f-6d17-4042-90a4-4a7b2da484b2 service nova] Acquiring lock "ebd6c719-32ae-410c-a444-db7ad3f8a9c5-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 789.839914] env[67752]: DEBUG oslo_concurrency.lockutils [req-3280b973-52ef-41fa-940b-2e18c376b6ed req-650a9f3f-6d17-4042-90a4-4a7b2da484b2 service nova] Lock "ebd6c719-32ae-410c-a444-db7ad3f8a9c5-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 789.840144] env[67752]: DEBUG oslo_concurrency.lockutils [req-3280b973-52ef-41fa-940b-2e18c376b6ed req-650a9f3f-6d17-4042-90a4-4a7b2da484b2 service nova] Lock "ebd6c719-32ae-410c-a444-db7ad3f8a9c5-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 789.840655] env[67752]: DEBUG nova.compute.manager [req-3280b973-52ef-41fa-940b-2e18c376b6ed req-650a9f3f-6d17-4042-90a4-4a7b2da484b2 service nova] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] No waiting events found dispatching network-vif-plugged-8f7da60c-1ca7-47ee-b944-302dad820eec {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 789.840914] env[67752]: WARNING nova.compute.manager [req-3280b973-52ef-41fa-940b-2e18c376b6ed req-650a9f3f-6d17-4042-90a4-4a7b2da484b2 service nova] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Received unexpected event network-vif-plugged-8f7da60c-1ca7-47ee-b944-302dad820eec for instance with vm_state building and task_state spawning. [ 789.946334] env[67752]: DEBUG nova.network.neutron [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Updating instance_info_cache with network_info: [{"id": "8f7da60c-1ca7-47ee-b944-302dad820eec", "address": "fa:16:3e:8a:7c:c0", "network": {"id": "8754f810-4855-45bb-9a57-6c1511f0722f", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.77", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "c43e0a2618b34312898e0e4bb014b17c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "983826cf-6390-4ec6-bf97-30a1060947fc", "external-id": "nsx-vlan-transportzone-367", "segmentation_id": 367, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap8f7da60c-1c", "ovs_interfaceid": "8f7da60c-1ca7-47ee-b944-302dad820eec", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 789.958483] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Releasing lock "refresh_cache-ebd6c719-32ae-410c-a444-db7ad3f8a9c5" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 789.958817] env[67752]: DEBUG nova.compute.manager [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Instance network_info: |[{"id": "8f7da60c-1ca7-47ee-b944-302dad820eec", "address": "fa:16:3e:8a:7c:c0", "network": {"id": "8754f810-4855-45bb-9a57-6c1511f0722f", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.77", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "c43e0a2618b34312898e0e4bb014b17c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "983826cf-6390-4ec6-bf97-30a1060947fc", "external-id": "nsx-vlan-transportzone-367", "segmentation_id": 367, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap8f7da60c-1c", "ovs_interfaceid": "8f7da60c-1ca7-47ee-b944-302dad820eec", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 789.959408] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:8a:7c:c0', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '983826cf-6390-4ec6-bf97-30a1060947fc', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '8f7da60c-1ca7-47ee-b944-302dad820eec', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 789.968096] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Creating folder: Project (d0bab7da534b4669b94441f5b1330b2f). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 789.968096] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-f77fd9f0-4da2-40e1-99c2-31056455a119 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 789.978247] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Created folder: Project (d0bab7da534b4669b94441f5b1330b2f) in parent group-v639722. [ 789.978440] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Creating folder: Instances. Parent ref: group-v639769. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 789.978670] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-58085d44-f47c-430b-92e1-ba50afbf02a5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 789.987419] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Created folder: Instances in parent group-v639769. [ 789.987641] env[67752]: DEBUG oslo.service.loopingcall [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 789.987814] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 789.987998] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-4d0ac063-e04e-4f71-8347-1d22f3cd17f7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 790.006184] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 790.006184] env[67752]: value = "task-3199676" [ 790.006184] env[67752]: _type = "Task" [ 790.006184] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 790.017185] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199676, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 790.516362] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199676, 'name': CreateVM_Task, 'duration_secs': 0.30012} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 790.516654] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 790.517209] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 790.517376] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 790.517686] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 790.517920] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-d9d3c6e7-fc4e-472d-b6b0-7c949b721f2c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 790.522216] env[67752]: DEBUG oslo_vmware.api [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Waiting for the task: (returnval){ [ 790.522216] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]521a5ea6-8c1e-2a90-1d8d-e13231071c7d" [ 790.522216] env[67752]: _type = "Task" [ 790.522216] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 790.529473] env[67752]: DEBUG oslo_vmware.api [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]521a5ea6-8c1e-2a90-1d8d-e13231071c7d, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 791.033128] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 791.033340] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 791.033546] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 791.963831] env[67752]: DEBUG nova.compute.manager [req-a3a49ded-8e58-42d0-b5eb-fac640d9bca4 req-c4e6a1b2-cde5-428d-96ca-278c729350fb service nova] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Received event network-changed-8f7da60c-1ca7-47ee-b944-302dad820eec {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 791.964120] env[67752]: DEBUG nova.compute.manager [req-a3a49ded-8e58-42d0-b5eb-fac640d9bca4 req-c4e6a1b2-cde5-428d-96ca-278c729350fb service nova] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Refreshing instance network info cache due to event network-changed-8f7da60c-1ca7-47ee-b944-302dad820eec. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 791.964250] env[67752]: DEBUG oslo_concurrency.lockutils [req-a3a49ded-8e58-42d0-b5eb-fac640d9bca4 req-c4e6a1b2-cde5-428d-96ca-278c729350fb service nova] Acquiring lock "refresh_cache-ebd6c719-32ae-410c-a444-db7ad3f8a9c5" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 791.964393] env[67752]: DEBUG oslo_concurrency.lockutils [req-a3a49ded-8e58-42d0-b5eb-fac640d9bca4 req-c4e6a1b2-cde5-428d-96ca-278c729350fb service nova] Acquired lock "refresh_cache-ebd6c719-32ae-410c-a444-db7ad3f8a9c5" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 791.964553] env[67752]: DEBUG nova.network.neutron [req-a3a49ded-8e58-42d0-b5eb-fac640d9bca4 req-c4e6a1b2-cde5-428d-96ca-278c729350fb service nova] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Refreshing network info cache for port 8f7da60c-1ca7-47ee-b944-302dad820eec {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 792.443694] env[67752]: DEBUG nova.network.neutron [req-a3a49ded-8e58-42d0-b5eb-fac640d9bca4 req-c4e6a1b2-cde5-428d-96ca-278c729350fb service nova] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Updated VIF entry in instance network info cache for port 8f7da60c-1ca7-47ee-b944-302dad820eec. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 792.444138] env[67752]: DEBUG nova.network.neutron [req-a3a49ded-8e58-42d0-b5eb-fac640d9bca4 req-c4e6a1b2-cde5-428d-96ca-278c729350fb service nova] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Updating instance_info_cache with network_info: [{"id": "8f7da60c-1ca7-47ee-b944-302dad820eec", "address": "fa:16:3e:8a:7c:c0", "network": {"id": "8754f810-4855-45bb-9a57-6c1511f0722f", "bridge": "br-int", "label": "shared", "subnets": [{"cidr": "192.168.233.0/24", "dns": [], "gateway": {"address": "192.168.233.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.233.77", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.233.2"}}], "meta": {"injected": false, "tenant_id": "c43e0a2618b34312898e0e4bb014b17c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "983826cf-6390-4ec6-bf97-30a1060947fc", "external-id": "nsx-vlan-transportzone-367", "segmentation_id": 367, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap8f7da60c-1c", "ovs_interfaceid": "8f7da60c-1ca7-47ee-b944-302dad820eec", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 792.454079] env[67752]: DEBUG oslo_concurrency.lockutils [req-a3a49ded-8e58-42d0-b5eb-fac640d9bca4 req-c4e6a1b2-cde5-428d-96ca-278c729350fb service nova] Releasing lock "refresh_cache-ebd6c719-32ae-410c-a444-db7ad3f8a9c5" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 796.936955] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Acquiring lock "1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 796.938683] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Lock "1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 798.953527] env[67752]: DEBUG oslo_concurrency.lockutils [None req-03d3ac84-69f7-4407-a09a-9b81ccce1e86 tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Acquiring lock "39479042-b891-4f25-9992-47312c6b6b43" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 803.136310] env[67752]: DEBUG oslo_concurrency.lockutils [None req-47057ad5-2f30-40e2-a8a5-1711e6b7e520 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquiring lock "9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 804.406506] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e6527cba-2f5b-4125-8fd0-2ffac9f9cafa tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquiring lock "5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 808.286917] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e8016b21-7c3d-4f6a-9a56-cf803514cb24 tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Acquiring lock "eca7c94f-9bd1-4c15-988e-d4e33086ba40" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 809.090019] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9b490ea7-b680-4746-bf26-c85ace342ec3 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquiring lock "ff4895f8-0374-4b34-a5e8-d3ba24f840a5" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 809.823717] env[67752]: DEBUG oslo_concurrency.lockutils [None req-23bde4ab-c3d3-4b40-b827-1c117f98a602 tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Acquiring lock "ca18af6b-6b80-44f3-bd8b-f61e2596acd7" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 813.647845] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4afda383-bd2a-490e-9e57-c8e10d9bd133 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Acquiring lock "02942b29-2343-441e-9e2b-aca2af57c849" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 819.603181] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a62a8de1-5e3c-479b-b435-c99583f74b98 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Acquiring lock "ebd6c719-32ae-410c-a444-db7ad3f8a9c5" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 831.634761] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 831.635121] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Cleaning up deleted instances {{(pid=67752) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11219}} [ 831.655893] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] There are 1 instances to clean {{(pid=67752) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11228}} [ 831.656177] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: cbe3fe72-5a91-4c81-ac62-be6592aa806a] Instance has had 0 of 5 cleanup attempts {{(pid=67752) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11232}} [ 831.709047] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 831.709047] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Cleaning up deleted instances with incomplete migration {{(pid=67752) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11257}} [ 831.723302] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 834.716787] env[67752]: WARNING oslo_vmware.rw_handles [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 834.716787] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 834.716787] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 834.716787] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 834.716787] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 834.716787] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 834.716787] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 834.716787] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 834.716787] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 834.716787] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 834.716787] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 834.716787] env[67752]: ERROR oslo_vmware.rw_handles [ 834.717386] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/b9454410-fb34-45cb-b3b4-032dadc5f7b9/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 834.719134] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 834.719392] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Copying Virtual Disk [datastore2] vmware_temp/b9454410-fb34-45cb-b3b4-032dadc5f7b9/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/b9454410-fb34-45cb-b3b4-032dadc5f7b9/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 834.719692] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-0bebb1c3-65f4-475c-b20b-ba7622b0812b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 834.726225] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 834.729734] env[67752]: DEBUG oslo_vmware.api [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Waiting for the task: (returnval){ [ 834.729734] env[67752]: value = "task-3199677" [ 834.729734] env[67752]: _type = "Task" [ 834.729734] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 834.738631] env[67752]: DEBUG oslo_vmware.api [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Task: {'id': task-3199677, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 835.238819] env[67752]: DEBUG oslo_vmware.exceptions [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 835.238819] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 835.239816] env[67752]: ERROR nova.compute.manager [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 835.239816] env[67752]: Faults: ['InvalidArgument'] [ 835.239816] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Traceback (most recent call last): [ 835.239816] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 835.239816] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] yield resources [ 835.239816] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 835.239816] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] self.driver.spawn(context, instance, image_meta, [ 835.239816] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 835.239816] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 835.239816] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 835.239816] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] self._fetch_image_if_missing(context, vi) [ 835.239816] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 835.240863] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] image_cache(vi, tmp_image_ds_loc) [ 835.240863] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 835.240863] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] vm_util.copy_virtual_disk( [ 835.240863] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 835.240863] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] session._wait_for_task(vmdk_copy_task) [ 835.240863] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 835.240863] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] return self.wait_for_task(task_ref) [ 835.240863] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 835.240863] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] return evt.wait() [ 835.240863] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 835.240863] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] result = hub.switch() [ 835.240863] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 835.240863] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] return self.greenlet.switch() [ 835.241447] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 835.241447] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] self.f(*self.args, **self.kw) [ 835.241447] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 835.241447] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] raise exceptions.translate_fault(task_info.error) [ 835.241447] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 835.241447] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Faults: ['InvalidArgument'] [ 835.241447] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] [ 835.241447] env[67752]: INFO nova.compute.manager [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Terminating instance [ 835.241847] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 835.244742] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 835.244742] env[67752]: DEBUG nova.compute.manager [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 835.244742] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 835.244742] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-baea0e7d-17cb-4b7e-845c-2f7fa849f24b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.245872] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-46be86aa-6426-4624-81b8-bb01678e42f2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.254527] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 835.254658] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-4f8002d8-2eee-4f21-a499-2163b099cae6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.258353] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 835.258477] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 835.259772] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-4ce66f75-87a6-438d-b74a-1e8781811d24 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.267136] env[67752]: DEBUG oslo_vmware.api [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Waiting for the task: (returnval){ [ 835.267136] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5251a109-37ea-fe92-92f3-e5b4546390a7" [ 835.267136] env[67752]: _type = "Task" [ 835.267136] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 835.276752] env[67752]: DEBUG oslo_vmware.api [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5251a109-37ea-fe92-92f3-e5b4546390a7, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 835.318399] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 835.318638] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 835.318821] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Deleting the datastore file [datastore2] fc4da5fa-1921-44e3-9100-042391d4ccb9 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 835.319107] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-0093626a-000c-4186-92cb-528ecd387c75 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.326682] env[67752]: DEBUG oslo_vmware.api [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Waiting for the task: (returnval){ [ 835.326682] env[67752]: value = "task-3199679" [ 835.326682] env[67752]: _type = "Task" [ 835.326682] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 835.335569] env[67752]: DEBUG oslo_vmware.api [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Task: {'id': task-3199679, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 835.634527] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 835.634715] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 835.634839] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 835.658120] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 835.658312] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 835.658450] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 835.658577] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 835.658704] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 835.658827] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 835.658950] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 835.659083] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 835.659208] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 835.659327] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 835.659449] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 835.659943] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 835.660142] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 835.660280] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 835.777411] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 835.777719] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Creating directory with path [datastore2] vmware_temp/4916e8f9-b1ba-4272-9a00-c563e628c44d/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 835.777899] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-ee4db229-c985-426f-9bca-ce32a92571c1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.790633] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Created directory with path [datastore2] vmware_temp/4916e8f9-b1ba-4272-9a00-c563e628c44d/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 835.790845] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Fetch image to [datastore2] vmware_temp/4916e8f9-b1ba-4272-9a00-c563e628c44d/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 835.791026] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/4916e8f9-b1ba-4272-9a00-c563e628c44d/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 835.791808] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6760531-d343-4188-a1dc-edac4c2c3586 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.798939] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a092005c-6cc3-4827-8446-37acd1294c05 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.808438] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c7dec92-bdd8-4981-b864-214fc4e07dbb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.847439] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-045fb2c4-aed7-434f-b0fc-01e294141276 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.856651] env[67752]: DEBUG oslo_vmware.api [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Task: {'id': task-3199679, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.086551} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 835.858203] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 835.858401] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 835.858569] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 835.858738] env[67752]: INFO nova.compute.manager [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Took 0.62 seconds to destroy the instance on the hypervisor. [ 835.860694] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-a3d2f82e-0c52-4d76-a065-7b160ce2f329 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 835.862972] env[67752]: DEBUG nova.compute.claims [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 835.863173] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 835.863480] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 835.885661] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 835.955291] env[67752]: DEBUG oslo_vmware.rw_handles [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/4916e8f9-b1ba-4272-9a00-c563e628c44d/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 836.024536] env[67752]: DEBUG oslo_vmware.rw_handles [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 836.024755] env[67752]: DEBUG oslo_vmware.rw_handles [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/4916e8f9-b1ba-4272-9a00-c563e628c44d/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 836.448801] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-da6ed21e-9c20-42df-b872-dbb35c98385f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 836.457501] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8cdd2600-34d8-4f35-84dc-d8b5c0078d63 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 836.491630] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93bbcef9-4e42-407f-9305-24ea83376e53 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 836.499747] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f141729a-084f-433c-a6ea-30fff1e6cbef {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 836.518072] env[67752]: DEBUG nova.compute.provider_tree [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 836.527866] env[67752]: DEBUG nova.scheduler.client.report [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 836.556251] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.691s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 836.556251] env[67752]: ERROR nova.compute.manager [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 836.556251] env[67752]: Faults: ['InvalidArgument'] [ 836.556251] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Traceback (most recent call last): [ 836.556251] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 836.556251] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] self.driver.spawn(context, instance, image_meta, [ 836.556251] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 836.556251] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 836.556251] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 836.556251] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] self._fetch_image_if_missing(context, vi) [ 836.556529] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 836.556529] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] image_cache(vi, tmp_image_ds_loc) [ 836.556529] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 836.556529] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] vm_util.copy_virtual_disk( [ 836.556529] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 836.556529] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] session._wait_for_task(vmdk_copy_task) [ 836.556529] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 836.556529] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] return self.wait_for_task(task_ref) [ 836.556529] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 836.556529] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] return evt.wait() [ 836.556529] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 836.556529] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] result = hub.switch() [ 836.556529] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 836.556888] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] return self.greenlet.switch() [ 836.556888] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 836.556888] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] self.f(*self.args, **self.kw) [ 836.556888] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 836.556888] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] raise exceptions.translate_fault(task_info.error) [ 836.556888] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 836.556888] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Faults: ['InvalidArgument'] [ 836.556888] env[67752]: ERROR nova.compute.manager [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] [ 836.556888] env[67752]: DEBUG nova.compute.utils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 836.557292] env[67752]: DEBUG nova.compute.manager [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Build of instance fc4da5fa-1921-44e3-9100-042391d4ccb9 was re-scheduled: A specified parameter was not correct: fileType [ 836.557292] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 836.557702] env[67752]: DEBUG nova.compute.manager [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 836.557885] env[67752]: DEBUG nova.compute.manager [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 836.558070] env[67752]: DEBUG nova.compute.manager [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 836.558241] env[67752]: DEBUG nova.network.neutron [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 836.634527] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 836.634748] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 837.635448] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 837.635938] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 837.652787] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 837.653056] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 837.653291] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 837.653491] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 837.655190] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aed81679-2e74-4e65-997e-dad7103c1496 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 837.664555] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-415aee48-684f-4fff-94e8-61f5a3bd947f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 837.680708] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-987597a6-a4c0-4c78-9f5d-48ad8d1fff9c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 837.687280] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50799edd-ce97-44bd-9091-07ad238a5b60 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 837.721685] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181007MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 837.721873] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 837.722312] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 837.724561] env[67752]: DEBUG nova.network.neutron [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 837.738673] env[67752]: INFO nova.compute.manager [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Took 1.18 seconds to deallocate network for instance. [ 837.841602] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fc4da5fa-1921-44e3-9100-042391d4ccb9 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 837.842809] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 39479042-b891-4f25-9992-47312c6b6b43 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 837.842809] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 837.842809] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 837.842809] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b0cc1acd-0e33-4547-a43b-549b3de81728 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 837.843467] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance eca7c94f-9bd1-4c15-988e-d4e33086ba40 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 837.843467] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ff4895f8-0374-4b34-a5e8-d3ba24f840a5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 837.843467] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ca18af6b-6b80-44f3-bd8b-f61e2596acd7 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 837.843467] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 02942b29-2343-441e-9e2b-aca2af57c849 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 837.843576] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ebd6c719-32ae-410c-a444-db7ad3f8a9c5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 837.864434] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 837.884404] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 40aaf487-275a-4e75-ac6a-127371e73b2c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 837.902816] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d2104921-5ec1-4fb8-8034-52dcdc403a92 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 837.905288] env[67752]: INFO nova.scheduler.client.report [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Deleted allocations for instance fc4da5fa-1921-44e3-9100-042391d4ccb9 [ 837.922892] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 16f0124d-146a-442a-8738-65f562b928e8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 837.929127] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ed334166-a675-4b2e-a382-3d84769d43e5 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Lock "fc4da5fa-1921-44e3-9100-042391d4ccb9" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 249.271s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 837.930294] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5b74ab26-f6dc-41b2-8498-174d0fc093f2 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Lock "fc4da5fa-1921-44e3-9100-042391d4ccb9" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 48.917s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 837.930516] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5b74ab26-f6dc-41b2-8498-174d0fc093f2 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Acquiring lock "fc4da5fa-1921-44e3-9100-042391d4ccb9-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 837.930724] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5b74ab26-f6dc-41b2-8498-174d0fc093f2 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Lock "fc4da5fa-1921-44e3-9100-042391d4ccb9-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 837.931355] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5b74ab26-f6dc-41b2-8498-174d0fc093f2 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Lock "fc4da5fa-1921-44e3-9100-042391d4ccb9-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 837.933654] env[67752]: INFO nova.compute.manager [None req-5b74ab26-f6dc-41b2-8498-174d0fc093f2 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Terminating instance [ 837.935639] env[67752]: DEBUG nova.compute.manager [None req-5b74ab26-f6dc-41b2-8498-174d0fc093f2 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 837.935825] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-5b74ab26-f6dc-41b2-8498-174d0fc093f2 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 837.936313] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-6eeb9343-f567-464c-8218-3f51692b9f97 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 837.946518] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd7d60c5-8728-4046-bb8e-d737c4624526 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 837.970033] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 8503063e-2888-44c6-bc2a-e9c82be791e2 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 837.972531] env[67752]: DEBUG nova.compute.manager [None req-06ef1eed-1b6d-4546-8b73-0988ea7f027c tempest-ServerRescueTestJSON-960884533 tempest-ServerRescueTestJSON-960884533-project-member] [instance: f528dc15-1a86-4af5-ac45-536fe3dfe101] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 837.994689] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-5b74ab26-f6dc-41b2-8498-174d0fc093f2 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance fc4da5fa-1921-44e3-9100-042391d4ccb9 could not be found. [ 837.994914] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-5b74ab26-f6dc-41b2-8498-174d0fc093f2 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 837.995118] env[67752]: INFO nova.compute.manager [None req-5b74ab26-f6dc-41b2-8498-174d0fc093f2 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Took 0.06 seconds to destroy the instance on the hypervisor. [ 837.995376] env[67752]: DEBUG oslo.service.loopingcall [None req-5b74ab26-f6dc-41b2-8498-174d0fc093f2 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 837.996040] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 631002d2-c268-4a6a-9945-c0898b886b93 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 837.997409] env[67752]: DEBUG nova.compute.manager [-] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 837.997585] env[67752]: DEBUG nova.network.neutron [-] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 838.013581] env[67752]: DEBUG nova.compute.manager [None req-06ef1eed-1b6d-4546-8b73-0988ea7f027c tempest-ServerRescueTestJSON-960884533 tempest-ServerRescueTestJSON-960884533-project-member] [instance: f528dc15-1a86-4af5-ac45-536fe3dfe101] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 838.024250] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 67812e2e-8827-4368-9790-a909c5ebb56b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 838.048298] env[67752]: DEBUG nova.network.neutron [-] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 838.051500] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 5ed08031-0ef8-4766-951a-37b315c55dc7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 838.051881] env[67752]: DEBUG oslo_concurrency.lockutils [None req-06ef1eed-1b6d-4546-8b73-0988ea7f027c tempest-ServerRescueTestJSON-960884533 tempest-ServerRescueTestJSON-960884533-project-member] Lock "f528dc15-1a86-4af5-ac45-536fe3dfe101" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 213.163s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 838.057919] env[67752]: INFO nova.compute.manager [-] [instance: fc4da5fa-1921-44e3-9100-042391d4ccb9] Took 0.06 seconds to deallocate network for instance. [ 838.069029] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 436d243a-d6d9-4958-94a8-7b09f5d9fdf3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 192, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 838.071060] env[67752]: DEBUG nova.compute.manager [None req-13c59bfe-ce4d-4478-876f-40c71e3a478f tempest-ServerDiagnosticsTest-1513525778 tempest-ServerDiagnosticsTest-1513525778-project-member] [instance: 4c0873a7-b8e5-4d3e-bdc0-3d236e3b9df6] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 838.089812] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f8b03251-6f23-4533-a917-f3fdac87888b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 838.104936] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 58d3b02f-3aac-4beb-a111-44f50c7f1bc8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 838.118762] env[67752]: DEBUG nova.compute.manager [None req-13c59bfe-ce4d-4478-876f-40c71e3a478f tempest-ServerDiagnosticsTest-1513525778 tempest-ServerDiagnosticsTest-1513525778-project-member] [instance: 4c0873a7-b8e5-4d3e-bdc0-3d236e3b9df6] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 838.135211] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 53eb8b0c-aa50-4d7f-8027-2725c815f134 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 838.152415] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 411bafb7-ef72-4529-b3f3-0b4955f23788 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 838.167347] env[67752]: DEBUG oslo_concurrency.lockutils [None req-13c59bfe-ce4d-4478-876f-40c71e3a478f tempest-ServerDiagnosticsTest-1513525778 tempest-ServerDiagnosticsTest-1513525778-project-member] Lock "4c0873a7-b8e5-4d3e-bdc0-3d236e3b9df6" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 206.866s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 838.171544] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ada70883-9bef-496e-a65f-ca84f6ee6211 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 838.182645] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 62f05780-7639-4a4f-975f-3f0863118c99 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 838.193227] env[67752]: DEBUG nova.compute.manager [None req-4e50a843-ce43-49a2-960c-709c4cf9a01e tempest-ServersV294TestFqdnHostnames-724274089 tempest-ServersV294TestFqdnHostnames-724274089-project-member] [instance: e3c2e1cf-cae5-474f-9f61-96d79815d846] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 838.205872] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 838.206147] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 9 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 838.206309] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1664MB phys_disk=200GB used_disk=9GB total_vcpus=48 used_vcpus=9 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 838.221093] env[67752]: DEBUG nova.compute.manager [None req-4e50a843-ce43-49a2-960c-709c4cf9a01e tempest-ServersV294TestFqdnHostnames-724274089 tempest-ServersV294TestFqdnHostnames-724274089-project-member] [instance: e3c2e1cf-cae5-474f-9f61-96d79815d846] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 838.248823] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5b74ab26-f6dc-41b2-8498-174d0fc093f2 tempest-ServerExternalEventsTest-1390465456 tempest-ServerExternalEventsTest-1390465456-project-member] Lock "fc4da5fa-1921-44e3-9100-042391d4ccb9" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.318s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 838.257480] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4e50a843-ce43-49a2-960c-709c4cf9a01e tempest-ServersV294TestFqdnHostnames-724274089 tempest-ServersV294TestFqdnHostnames-724274089-project-member] Lock "e3c2e1cf-cae5-474f-9f61-96d79815d846" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 205.961s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 838.268470] env[67752]: DEBUG nova.compute.manager [None req-b45c275f-58d0-4291-a4d3-b0aa404c21cf tempest-ServerActionsTestOtherB-2033325558 tempest-ServerActionsTestOtherB-2033325558-project-member] [instance: f22bcbae-0dcb-4666-b5e7-00c560e016c3] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 838.303738] env[67752]: DEBUG nova.compute.manager [None req-b45c275f-58d0-4291-a4d3-b0aa404c21cf tempest-ServerActionsTestOtherB-2033325558 tempest-ServerActionsTestOtherB-2033325558-project-member] [instance: f22bcbae-0dcb-4666-b5e7-00c560e016c3] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 838.329141] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b45c275f-58d0-4291-a4d3-b0aa404c21cf tempest-ServerActionsTestOtherB-2033325558 tempest-ServerActionsTestOtherB-2033325558-project-member] Lock "f22bcbae-0dcb-4666-b5e7-00c560e016c3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 204.377s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 838.347331] env[67752]: DEBUG nova.compute.manager [None req-29de0487-2264-4436-8ac3-d6e9fd25d33f tempest-ServerTagsTestJSON-923615097 tempest-ServerTagsTestJSON-923615097-project-member] [instance: c38268b5-b35d-4d8d-aa03-72100e2889f7] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 838.377796] env[67752]: DEBUG nova.compute.manager [None req-29de0487-2264-4436-8ac3-d6e9fd25d33f tempest-ServerTagsTestJSON-923615097 tempest-ServerTagsTestJSON-923615097-project-member] [instance: c38268b5-b35d-4d8d-aa03-72100e2889f7] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 838.404627] env[67752]: DEBUG oslo_concurrency.lockutils [None req-29de0487-2264-4436-8ac3-d6e9fd25d33f tempest-ServerTagsTestJSON-923615097 tempest-ServerTagsTestJSON-923615097-project-member] Lock "c38268b5-b35d-4d8d-aa03-72100e2889f7" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 201.752s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 838.419669] env[67752]: DEBUG nova.compute.manager [None req-c68aa49d-9279-43b1-b3e4-603b2f38fe16 tempest-AttachVolumeShelveTestJSON-812146230 tempest-AttachVolumeShelveTestJSON-812146230-project-member] [instance: c6a7f89e-d842-4b34-af36-ca60f12bcddd] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 838.455285] env[67752]: DEBUG nova.compute.manager [None req-c68aa49d-9279-43b1-b3e4-603b2f38fe16 tempest-AttachVolumeShelveTestJSON-812146230 tempest-AttachVolumeShelveTestJSON-812146230-project-member] [instance: c6a7f89e-d842-4b34-af36-ca60f12bcddd] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 838.476872] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c68aa49d-9279-43b1-b3e4-603b2f38fe16 tempest-AttachVolumeShelveTestJSON-812146230 tempest-AttachVolumeShelveTestJSON-812146230-project-member] Lock "c6a7f89e-d842-4b34-af36-ca60f12bcddd" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 201.735s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 838.488398] env[67752]: DEBUG nova.compute.manager [None req-2629dfa5-b887-4a2c-bf98-7ac2918140fd tempest-InstanceActionsTestJSON-1707315631 tempest-InstanceActionsTestJSON-1707315631-project-member] [instance: fa9cdf6d-032f-4069-a78e-fb16b4d2c457] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 838.514637] env[67752]: DEBUG nova.compute.manager [None req-2629dfa5-b887-4a2c-bf98-7ac2918140fd tempest-InstanceActionsTestJSON-1707315631 tempest-InstanceActionsTestJSON-1707315631-project-member] [instance: fa9cdf6d-032f-4069-a78e-fb16b4d2c457] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 838.541179] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2629dfa5-b887-4a2c-bf98-7ac2918140fd tempest-InstanceActionsTestJSON-1707315631 tempest-InstanceActionsTestJSON-1707315631-project-member] Lock "fa9cdf6d-032f-4069-a78e-fb16b4d2c457" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 200.115s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 838.551348] env[67752]: DEBUG nova.compute.manager [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 838.617413] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 838.624076] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8ec20828-c133-4ea2-bf4f-088d464db864 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 838.632119] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c1500487-cfd7-487b-b1a9-31ee6fcc7415 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 838.666997] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-12a8b94e-27e2-41d6-870d-27e332c5fc34 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 838.675684] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a72d3e5f-30a7-4b5f-afa3-d64497c779e6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 838.691170] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 838.701752] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 838.727699] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 838.727699] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 1.005s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 838.727839] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.111s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 838.732437] env[67752]: INFO nova.compute.claims [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 839.250567] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-afa6815e-cb9d-4368-9544-d4544b5139d1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.259073] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-543d9ee0-3c6c-4454-a528-455585570be8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.292978] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a77ae631-bcf9-48d0-b71f-5f14b8f125f0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.300585] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21f449f3-1291-44a6-b994-d8faaac5d3f4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.315675] env[67752]: DEBUG nova.compute.provider_tree [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 839.328888] env[67752]: DEBUG nova.scheduler.client.report [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 839.349317] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.621s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 839.349829] env[67752]: DEBUG nova.compute.manager [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 839.403167] env[67752]: DEBUG nova.compute.utils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 839.404595] env[67752]: DEBUG nova.compute.manager [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 839.404782] env[67752]: DEBUG nova.network.neutron [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 839.426334] env[67752]: DEBUG nova.compute.manager [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 839.498993] env[67752]: DEBUG nova.compute.manager [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 839.530024] env[67752]: DEBUG nova.virt.hardware [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 839.530346] env[67752]: DEBUG nova.virt.hardware [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 839.530549] env[67752]: DEBUG nova.virt.hardware [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 839.530771] env[67752]: DEBUG nova.virt.hardware [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 839.530998] env[67752]: DEBUG nova.virt.hardware [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 839.531166] env[67752]: DEBUG nova.virt.hardware [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 839.532041] env[67752]: DEBUG nova.virt.hardware [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 839.532041] env[67752]: DEBUG nova.virt.hardware [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 839.532041] env[67752]: DEBUG nova.virt.hardware [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 839.532041] env[67752]: DEBUG nova.virt.hardware [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 839.532264] env[67752]: DEBUG nova.virt.hardware [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 839.533127] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d05622d-10c0-469b-b3c8-03c02cfc0840 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.543107] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d5079ef4-a4e6-4db7-a1c7-6484ce3baaa1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 839.627720] env[67752]: DEBUG nova.policy [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'b66e0b6b65064e48be600cfb1b7249a8', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '1481c8be1354466296d78d4ad7aad472', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 840.164771] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d64f589f-61c3-42d5-84b9-e96830929821 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Acquiring lock "580c2aa9-160e-40ec-a1e1-0bbaaf60bca9" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 840.734556] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 840.954957] env[67752]: DEBUG nova.network.neutron [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Successfully created port: a65f35a5-6d58-4f62-af67-ffa88023f39c {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 841.413415] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Acquiring lock "b271cd73-a148-48d6-bed9-6a99512457bd" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 841.413783] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Lock "b271cd73-a148-48d6-bed9-6a99512457bd" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 842.614013] env[67752]: DEBUG nova.network.neutron [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Successfully updated port: a65f35a5-6d58-4f62-af67-ffa88023f39c {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 842.628040] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Acquiring lock "refresh_cache-580c2aa9-160e-40ec-a1e1-0bbaaf60bca9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 842.628266] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Acquired lock "refresh_cache-580c2aa9-160e-40ec-a1e1-0bbaaf60bca9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 842.628431] env[67752]: DEBUG nova.network.neutron [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 842.689439] env[67752]: DEBUG nova.network.neutron [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 843.077379] env[67752]: DEBUG nova.network.neutron [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Updating instance_info_cache with network_info: [{"id": "a65f35a5-6d58-4f62-af67-ffa88023f39c", "address": "fa:16:3e:5c:ab:c5", "network": {"id": "d34d91ef-0a56-4297-a99a-4e7cd0b41bb0", "bridge": "br-int", "label": "tempest-ImagesNegativeTestJSON-1785538206-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.10", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "1481c8be1354466296d78d4ad7aad472", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "753b69c3-4ef5-44a1-80cf-eab4edbe0dd5", "external-id": "nsx-vlan-transportzone-878", "segmentation_id": 878, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapa65f35a5-6d", "ovs_interfaceid": "a65f35a5-6d58-4f62-af67-ffa88023f39c", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 843.099736] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Releasing lock "refresh_cache-580c2aa9-160e-40ec-a1e1-0bbaaf60bca9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 843.100247] env[67752]: DEBUG nova.compute.manager [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Instance network_info: |[{"id": "a65f35a5-6d58-4f62-af67-ffa88023f39c", "address": "fa:16:3e:5c:ab:c5", "network": {"id": "d34d91ef-0a56-4297-a99a-4e7cd0b41bb0", "bridge": "br-int", "label": "tempest-ImagesNegativeTestJSON-1785538206-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.10", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "1481c8be1354466296d78d4ad7aad472", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "753b69c3-4ef5-44a1-80cf-eab4edbe0dd5", "external-id": "nsx-vlan-transportzone-878", "segmentation_id": 878, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapa65f35a5-6d", "ovs_interfaceid": "a65f35a5-6d58-4f62-af67-ffa88023f39c", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 843.100540] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:5c:ab:c5', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '753b69c3-4ef5-44a1-80cf-eab4edbe0dd5', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'a65f35a5-6d58-4f62-af67-ffa88023f39c', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 843.108158] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Creating folder: Project (1481c8be1354466296d78d4ad7aad472). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 843.108775] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-a3d81f0d-49da-49e4-9dc8-f92bb948adb5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 843.120189] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Created folder: Project (1481c8be1354466296d78d4ad7aad472) in parent group-v639722. [ 843.120461] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Creating folder: Instances. Parent ref: group-v639772. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 843.120758] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-384558e5-fd43-4918-a4ba-f57c1ef3134a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 843.129255] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Created folder: Instances in parent group-v639772. [ 843.129522] env[67752]: DEBUG oslo.service.loopingcall [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 843.129790] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 843.130101] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-7bddeb2b-c0a8-4f37-9078-23b09dd23445 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 843.152081] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 843.152081] env[67752]: value = "task-3199682" [ 843.152081] env[67752]: _type = "Task" [ 843.152081] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 843.163041] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199682, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 843.638451] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b416a339-6230-4165-9968-d38a5aabc674 tempest-AttachVolumeShelveTestJSON-812146230 tempest-AttachVolumeShelveTestJSON-812146230-project-member] Acquiring lock "fe967117-5b16-4b5e-aeb7-626e78fee756" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 843.639181] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b416a339-6230-4165-9968-d38a5aabc674 tempest-AttachVolumeShelveTestJSON-812146230 tempest-AttachVolumeShelveTestJSON-812146230-project-member] Lock "fe967117-5b16-4b5e-aeb7-626e78fee756" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 843.665198] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199682, 'name': CreateVM_Task, 'duration_secs': 0.319686} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 843.665455] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 843.666251] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 843.666563] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 843.667016] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 843.667306] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-92b56320-60a6-4270-8720-e9274c6650d7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 843.673042] env[67752]: DEBUG nova.compute.manager [req-23d69a6c-7049-49f9-84fb-e7cec4866152 req-7e07a762-8b3c-45b4-9592-ca7c3f4cb377 service nova] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Received event network-vif-plugged-a65f35a5-6d58-4f62-af67-ffa88023f39c {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 843.673885] env[67752]: DEBUG oslo_concurrency.lockutils [req-23d69a6c-7049-49f9-84fb-e7cec4866152 req-7e07a762-8b3c-45b4-9592-ca7c3f4cb377 service nova] Acquiring lock "580c2aa9-160e-40ec-a1e1-0bbaaf60bca9-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 843.674767] env[67752]: DEBUG oslo_concurrency.lockutils [req-23d69a6c-7049-49f9-84fb-e7cec4866152 req-7e07a762-8b3c-45b4-9592-ca7c3f4cb377 service nova] Lock "580c2aa9-160e-40ec-a1e1-0bbaaf60bca9-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 843.675028] env[67752]: DEBUG oslo_concurrency.lockutils [req-23d69a6c-7049-49f9-84fb-e7cec4866152 req-7e07a762-8b3c-45b4-9592-ca7c3f4cb377 service nova] Lock "580c2aa9-160e-40ec-a1e1-0bbaaf60bca9-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 843.675274] env[67752]: DEBUG nova.compute.manager [req-23d69a6c-7049-49f9-84fb-e7cec4866152 req-7e07a762-8b3c-45b4-9592-ca7c3f4cb377 service nova] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] No waiting events found dispatching network-vif-plugged-a65f35a5-6d58-4f62-af67-ffa88023f39c {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 843.675546] env[67752]: WARNING nova.compute.manager [req-23d69a6c-7049-49f9-84fb-e7cec4866152 req-7e07a762-8b3c-45b4-9592-ca7c3f4cb377 service nova] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Received unexpected event network-vif-plugged-a65f35a5-6d58-4f62-af67-ffa88023f39c for instance with vm_state building and task_state deleting. [ 843.677474] env[67752]: DEBUG oslo_vmware.api [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Waiting for the task: (returnval){ [ 843.677474] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]525bee6d-b7a9-92e1-7694-faebe6b0bcca" [ 843.677474] env[67752]: _type = "Task" [ 843.677474] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 843.687837] env[67752]: DEBUG oslo_vmware.api [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]525bee6d-b7a9-92e1-7694-faebe6b0bcca, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 844.190780] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 844.191061] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 844.192336] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 846.288050] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6141b860-2aa0-406b-b112-0dc765b9153a tempest-ServersNegativeTestJSON-289807960 tempest-ServersNegativeTestJSON-289807960-project-member] Acquiring lock "07fe1ba8-e5c2-421f-98db-680d90f39913" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 846.288387] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6141b860-2aa0-406b-b112-0dc765b9153a tempest-ServersNegativeTestJSON-289807960 tempest-ServersNegativeTestJSON-289807960-project-member] Lock "07fe1ba8-e5c2-421f-98db-680d90f39913" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 846.369767] env[67752]: DEBUG nova.compute.manager [req-3fbf3897-4a0f-449d-98ae-d51d68d11375 req-30017151-28e2-4c18-ad4f-bdc7391e5eac service nova] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Received event network-changed-a65f35a5-6d58-4f62-af67-ffa88023f39c {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 846.370434] env[67752]: DEBUG nova.compute.manager [req-3fbf3897-4a0f-449d-98ae-d51d68d11375 req-30017151-28e2-4c18-ad4f-bdc7391e5eac service nova] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Refreshing instance network info cache due to event network-changed-a65f35a5-6d58-4f62-af67-ffa88023f39c. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 846.371648] env[67752]: DEBUG oslo_concurrency.lockutils [req-3fbf3897-4a0f-449d-98ae-d51d68d11375 req-30017151-28e2-4c18-ad4f-bdc7391e5eac service nova] Acquiring lock "refresh_cache-580c2aa9-160e-40ec-a1e1-0bbaaf60bca9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 846.371860] env[67752]: DEBUG oslo_concurrency.lockutils [req-3fbf3897-4a0f-449d-98ae-d51d68d11375 req-30017151-28e2-4c18-ad4f-bdc7391e5eac service nova] Acquired lock "refresh_cache-580c2aa9-160e-40ec-a1e1-0bbaaf60bca9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 846.372864] env[67752]: DEBUG nova.network.neutron [req-3fbf3897-4a0f-449d-98ae-d51d68d11375 req-30017151-28e2-4c18-ad4f-bdc7391e5eac service nova] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Refreshing network info cache for port a65f35a5-6d58-4f62-af67-ffa88023f39c {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 846.927929] env[67752]: DEBUG nova.network.neutron [req-3fbf3897-4a0f-449d-98ae-d51d68d11375 req-30017151-28e2-4c18-ad4f-bdc7391e5eac service nova] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Updated VIF entry in instance network info cache for port a65f35a5-6d58-4f62-af67-ffa88023f39c. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 846.928321] env[67752]: DEBUG nova.network.neutron [req-3fbf3897-4a0f-449d-98ae-d51d68d11375 req-30017151-28e2-4c18-ad4f-bdc7391e5eac service nova] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Updating instance_info_cache with network_info: [{"id": "a65f35a5-6d58-4f62-af67-ffa88023f39c", "address": "fa:16:3e:5c:ab:c5", "network": {"id": "d34d91ef-0a56-4297-a99a-4e7cd0b41bb0", "bridge": "br-int", "label": "tempest-ImagesNegativeTestJSON-1785538206-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.10", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "1481c8be1354466296d78d4ad7aad472", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "753b69c3-4ef5-44a1-80cf-eab4edbe0dd5", "external-id": "nsx-vlan-transportzone-878", "segmentation_id": 878, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapa65f35a5-6d", "ovs_interfaceid": "a65f35a5-6d58-4f62-af67-ffa88023f39c", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 846.943600] env[67752]: DEBUG oslo_concurrency.lockutils [req-3fbf3897-4a0f-449d-98ae-d51d68d11375 req-30017151-28e2-4c18-ad4f-bdc7391e5eac service nova] Releasing lock "refresh_cache-580c2aa9-160e-40ec-a1e1-0bbaaf60bca9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 847.209020] env[67752]: DEBUG oslo_concurrency.lockutils [None req-cd94167a-d25b-4a92-b001-dd91b9582577 tempest-ServerDiagnosticsNegativeTest-355792777 tempest-ServerDiagnosticsNegativeTest-355792777-project-member] Acquiring lock "a3a99c59-0c99-424e-b86c-5f58b494b4ef" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 847.209512] env[67752]: DEBUG oslo_concurrency.lockutils [None req-cd94167a-d25b-4a92-b001-dd91b9582577 tempest-ServerDiagnosticsNegativeTest-355792777 tempest-ServerDiagnosticsNegativeTest-355792777-project-member] Lock "a3a99c59-0c99-424e-b86c-5f58b494b4ef" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 850.629482] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbfd8929-3e44-4655-9216-fd5ca59163b4 tempest-ServerGroupTestJSON-598670425 tempest-ServerGroupTestJSON-598670425-project-member] Acquiring lock "505f538c-051f-42e2-9da0-057e29bbb009" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 850.629482] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbfd8929-3e44-4655-9216-fd5ca59163b4 tempest-ServerGroupTestJSON-598670425 tempest-ServerGroupTestJSON-598670425-project-member] Lock "505f538c-051f-42e2-9da0-057e29bbb009" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 851.208880] env[67752]: DEBUG oslo_concurrency.lockutils [None req-39ef8043-c1d5-4e53-8ea1-e3c88ba7ee9f tempest-ServerShowV247Test-1751066233 tempest-ServerShowV247Test-1751066233-project-member] Acquiring lock "a4bd5964-ee9e-44dc-a691-2382dff58c36" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 851.209125] env[67752]: DEBUG oslo_concurrency.lockutils [None req-39ef8043-c1d5-4e53-8ea1-e3c88ba7ee9f tempest-ServerShowV247Test-1751066233 tempest-ServerShowV247Test-1751066233-project-member] Lock "a4bd5964-ee9e-44dc-a691-2382dff58c36" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 852.198570] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e5c22520-22ec-47ed-9010-411eb42dd3f7 tempest-ServerShowV247Test-1751066233 tempest-ServerShowV247Test-1751066233-project-member] Acquiring lock "b800574b-cf7f-458b-ad5c-7c5c3dc5044f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 852.198570] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e5c22520-22ec-47ed-9010-411eb42dd3f7 tempest-ServerShowV247Test-1751066233 tempest-ServerShowV247Test-1751066233-project-member] Lock "b800574b-cf7f-458b-ad5c-7c5c3dc5044f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 853.193888] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8631916d-5152-413b-96d2-ef7bb793d025 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Acquiring lock "ee2343dc-52da-4327-afb2-83a774cb6fd0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 853.194173] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8631916d-5152-413b-96d2-ef7bb793d025 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Lock "ee2343dc-52da-4327-afb2-83a774cb6fd0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 857.164627] env[67752]: DEBUG oslo_concurrency.lockutils [None req-cd49a92d-3ef3-44dd-82e4-e594546ff1c4 tempest-SecurityGroupsTestJSON-1418396030 tempest-SecurityGroupsTestJSON-1418396030-project-member] Acquiring lock "017d212b-25f6-4b00-a1f4-090077a1fe72" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 857.165036] env[67752]: DEBUG oslo_concurrency.lockutils [None req-cd49a92d-3ef3-44dd-82e4-e594546ff1c4 tempest-SecurityGroupsTestJSON-1418396030 tempest-SecurityGroupsTestJSON-1418396030-project-member] Lock "017d212b-25f6-4b00-a1f4-090077a1fe72" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 858.126446] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d3bf8ccf-d395-49d2-9675-da151d59e62c tempest-ServersAaction247Test-556426769 tempest-ServersAaction247Test-556426769-project-member] Acquiring lock "15a38322-5e08-4914-b609-a9b5941c9253" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 858.126902] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d3bf8ccf-d395-49d2-9675-da151d59e62c tempest-ServersAaction247Test-556426769 tempest-ServersAaction247Test-556426769-project-member] Lock "15a38322-5e08-4914-b609-a9b5941c9253" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 863.254942] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c753e970-12bd-4719-b442-113eb87194b3 tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] Acquiring lock "0a502eb7-d333-4a4c-a75b-710bf7921871" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 863.255247] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c753e970-12bd-4719-b442-113eb87194b3 tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] Lock "0a502eb7-d333-4a4c-a75b-710bf7921871" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 873.159176] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c5206e6-bc4f-4023-9741-e1f496e47c50 tempest-ServerActionsTestOtherA-1579097132 tempest-ServerActionsTestOtherA-1579097132-project-member] Acquiring lock "9ad07396-2d8c-4277-b868-206e309631f0" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 873.159556] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c5206e6-bc4f-4023-9741-e1f496e47c50 tempest-ServerActionsTestOtherA-1579097132 tempest-ServerActionsTestOtherA-1579097132-project-member] Lock "9ad07396-2d8c-4277-b868-206e309631f0" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 884.738585] env[67752]: WARNING oslo_vmware.rw_handles [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 884.738585] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 884.738585] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 884.738585] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 884.738585] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 884.738585] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 884.738585] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 884.738585] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 884.738585] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 884.738585] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 884.738585] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 884.738585] env[67752]: ERROR oslo_vmware.rw_handles [ 884.738585] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/4916e8f9-b1ba-4272-9a00-c563e628c44d/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 884.740451] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 884.740451] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Copying Virtual Disk [datastore2] vmware_temp/4916e8f9-b1ba-4272-9a00-c563e628c44d/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/4916e8f9-b1ba-4272-9a00-c563e628c44d/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 884.741212] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-bc840bcc-2631-4c5c-91d0-e767e8328f68 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 884.748211] env[67752]: DEBUG oslo_vmware.api [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Waiting for the task: (returnval){ [ 884.748211] env[67752]: value = "task-3199683" [ 884.748211] env[67752]: _type = "Task" [ 884.748211] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 884.756805] env[67752]: DEBUG oslo_vmware.api [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Task: {'id': task-3199683, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 885.257899] env[67752]: DEBUG oslo_vmware.exceptions [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 885.258284] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 885.258871] env[67752]: ERROR nova.compute.manager [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 885.258871] env[67752]: Faults: ['InvalidArgument'] [ 885.258871] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] Traceback (most recent call last): [ 885.258871] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 885.258871] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] yield resources [ 885.258871] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 885.258871] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] self.driver.spawn(context, instance, image_meta, [ 885.258871] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 885.258871] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] self._vmops.spawn(context, instance, image_meta, injected_files, [ 885.258871] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 885.258871] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] self._fetch_image_if_missing(context, vi) [ 885.258871] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 885.259248] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] image_cache(vi, tmp_image_ds_loc) [ 885.259248] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 885.259248] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] vm_util.copy_virtual_disk( [ 885.259248] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 885.259248] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] session._wait_for_task(vmdk_copy_task) [ 885.259248] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 885.259248] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] return self.wait_for_task(task_ref) [ 885.259248] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 885.259248] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] return evt.wait() [ 885.259248] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 885.259248] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] result = hub.switch() [ 885.259248] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 885.259248] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] return self.greenlet.switch() [ 885.259613] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 885.259613] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] self.f(*self.args, **self.kw) [ 885.259613] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 885.259613] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] raise exceptions.translate_fault(task_info.error) [ 885.259613] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 885.259613] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] Faults: ['InvalidArgument'] [ 885.259613] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] [ 885.259613] env[67752]: INFO nova.compute.manager [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Terminating instance [ 885.261316] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 885.261569] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 885.261836] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-f3a127d9-90b6-43d8-86c1-8df2c12f8d2a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 885.264075] env[67752]: DEBUG nova.compute.manager [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 885.264317] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 885.265063] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-487100a3-828a-4461-bb26-03b9b26edf09 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 885.271497] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 885.271898] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-d33957d7-d0dc-4a21-987b-031aad5b6df4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 885.274034] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 885.274253] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 885.275202] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-187843a5-eee6-4d7c-85eb-fcff368120fb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 885.279872] env[67752]: DEBUG oslo_vmware.api [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Waiting for the task: (returnval){ [ 885.279872] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]525ee9b4-70a2-0631-fe9d-11b6f2737558" [ 885.279872] env[67752]: _type = "Task" [ 885.279872] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 885.287441] env[67752]: DEBUG oslo_vmware.api [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]525ee9b4-70a2-0631-fe9d-11b6f2737558, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 885.358248] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 885.358540] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 885.358818] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Deleting the datastore file [datastore2] 39479042-b891-4f25-9992-47312c6b6b43 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 885.359127] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-ef39f3d3-a21d-4637-9deb-ddede8a962bf {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 885.365914] env[67752]: DEBUG oslo_vmware.api [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Waiting for the task: (returnval){ [ 885.365914] env[67752]: value = "task-3199685" [ 885.365914] env[67752]: _type = "Task" [ 885.365914] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 885.374272] env[67752]: DEBUG oslo_vmware.api [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Task: {'id': task-3199685, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 885.790369] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 885.790659] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Creating directory with path [datastore2] vmware_temp/51d21b0f-797d-4881-989c-664ebcb407f9/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 885.790901] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-1f319447-2dc4-4dfe-a5cd-e0cfd32813c9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 885.802354] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Created directory with path [datastore2] vmware_temp/51d21b0f-797d-4881-989c-664ebcb407f9/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 885.802550] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Fetch image to [datastore2] vmware_temp/51d21b0f-797d-4881-989c-664ebcb407f9/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 885.802726] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/51d21b0f-797d-4881-989c-664ebcb407f9/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 885.803518] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cc933326-85c7-4cff-be42-b0c814bd99ae {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 885.810230] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-948702e3-aafe-4f3e-a579-96b5814d3c84 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 885.819198] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a6fb558e-eb69-4ebf-b56b-d96e8c18f481 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 885.850564] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-da88f5fd-43be-4298-99a4-cc0cf0b3c05e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 885.856344] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-72e52cdb-6d8b-4323-bb76-25c14fc1c8a6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 885.873461] env[67752]: DEBUG oslo_vmware.api [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Task: {'id': task-3199685, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.069182} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 885.873713] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 885.873895] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 885.874079] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 885.874263] env[67752]: INFO nova.compute.manager [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Took 0.61 seconds to destroy the instance on the hypervisor. [ 885.876333] env[67752]: DEBUG nova.compute.claims [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 885.876511] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 885.876728] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 885.955566] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 886.078630] env[67752]: DEBUG oslo_vmware.rw_handles [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/51d21b0f-797d-4881-989c-664ebcb407f9/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 886.144304] env[67752]: DEBUG oslo_vmware.rw_handles [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 886.144505] env[67752]: DEBUG oslo_vmware.rw_handles [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Closing write handle for https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/51d21b0f-797d-4881-989c-664ebcb407f9/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 886.301375] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c801b388-ab08-49ee-a5f1-f57bce992cf3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 886.308740] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2809449d-8047-4a41-86ef-985ced4dda3d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 886.337704] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7c3f5be5-5340-46c6-9503-7daec72c1a44 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 886.344283] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-567ce74c-6220-4c7c-9e49-69f4fdc5bb40 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 886.356947] env[67752]: DEBUG nova.compute.provider_tree [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 886.366090] env[67752]: DEBUG nova.scheduler.client.report [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 886.380897] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.504s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 886.381419] env[67752]: ERROR nova.compute.manager [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 886.381419] env[67752]: Faults: ['InvalidArgument'] [ 886.381419] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] Traceback (most recent call last): [ 886.381419] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 886.381419] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] self.driver.spawn(context, instance, image_meta, [ 886.381419] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 886.381419] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] self._vmops.spawn(context, instance, image_meta, injected_files, [ 886.381419] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 886.381419] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] self._fetch_image_if_missing(context, vi) [ 886.381419] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 886.381419] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] image_cache(vi, tmp_image_ds_loc) [ 886.381419] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 886.381763] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] vm_util.copy_virtual_disk( [ 886.381763] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 886.381763] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] session._wait_for_task(vmdk_copy_task) [ 886.381763] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 886.381763] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] return self.wait_for_task(task_ref) [ 886.381763] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 886.381763] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] return evt.wait() [ 886.381763] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 886.381763] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] result = hub.switch() [ 886.381763] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 886.381763] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] return self.greenlet.switch() [ 886.381763] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 886.381763] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] self.f(*self.args, **self.kw) [ 886.382116] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 886.382116] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] raise exceptions.translate_fault(task_info.error) [ 886.382116] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 886.382116] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] Faults: ['InvalidArgument'] [ 886.382116] env[67752]: ERROR nova.compute.manager [instance: 39479042-b891-4f25-9992-47312c6b6b43] [ 886.382116] env[67752]: DEBUG nova.compute.utils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 886.383442] env[67752]: DEBUG nova.compute.manager [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Build of instance 39479042-b891-4f25-9992-47312c6b6b43 was re-scheduled: A specified parameter was not correct: fileType [ 886.383442] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 886.383815] env[67752]: DEBUG nova.compute.manager [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 886.383986] env[67752]: DEBUG nova.compute.manager [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 886.384170] env[67752]: DEBUG nova.compute.manager [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 886.384332] env[67752]: DEBUG nova.network.neutron [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 886.869949] env[67752]: DEBUG nova.network.neutron [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 886.881927] env[67752]: INFO nova.compute.manager [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Took 0.50 seconds to deallocate network for instance. [ 886.991101] env[67752]: INFO nova.scheduler.client.report [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Deleted allocations for instance 39479042-b891-4f25-9992-47312c6b6b43 [ 887.019336] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5db1bc18-1752-437c-b931-ab7309edbf6f tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Lock "39479042-b891-4f25-9992-47312c6b6b43" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 286.234s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 887.019336] env[67752]: DEBUG oslo_concurrency.lockutils [None req-03d3ac84-69f7-4407-a09a-9b81ccce1e86 tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Lock "39479042-b891-4f25-9992-47312c6b6b43" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 88.069s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 887.019632] env[67752]: DEBUG oslo_concurrency.lockutils [None req-03d3ac84-69f7-4407-a09a-9b81ccce1e86 tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Acquiring lock "39479042-b891-4f25-9992-47312c6b6b43-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 887.019817] env[67752]: DEBUG oslo_concurrency.lockutils [None req-03d3ac84-69f7-4407-a09a-9b81ccce1e86 tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Lock "39479042-b891-4f25-9992-47312c6b6b43-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 887.019901] env[67752]: DEBUG oslo_concurrency.lockutils [None req-03d3ac84-69f7-4407-a09a-9b81ccce1e86 tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Lock "39479042-b891-4f25-9992-47312c6b6b43-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 887.021935] env[67752]: INFO nova.compute.manager [None req-03d3ac84-69f7-4407-a09a-9b81ccce1e86 tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Terminating instance [ 887.023614] env[67752]: DEBUG nova.compute.manager [None req-03d3ac84-69f7-4407-a09a-9b81ccce1e86 tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 887.023803] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-03d3ac84-69f7-4407-a09a-9b81ccce1e86 tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 887.024546] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5f5db80f-2612-46e7-af3b-271537c18465 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 887.033477] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c91c595-69ee-4b12-911e-80233c350b50 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 887.048493] env[67752]: DEBUG nova.compute.manager [None req-642b4fb0-db2a-4c92-bc8e-8b8564f22880 tempest-ServerPasswordTestJSON-1994449735 tempest-ServerPasswordTestJSON-1994449735-project-member] [instance: 40aaf487-275a-4e75-ac6a-127371e73b2c] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 887.061396] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-03d3ac84-69f7-4407-a09a-9b81ccce1e86 tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 39479042-b891-4f25-9992-47312c6b6b43 could not be found. [ 887.061614] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-03d3ac84-69f7-4407-a09a-9b81ccce1e86 tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 887.061773] env[67752]: INFO nova.compute.manager [None req-03d3ac84-69f7-4407-a09a-9b81ccce1e86 tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Took 0.04 seconds to destroy the instance on the hypervisor. [ 887.062828] env[67752]: DEBUG oslo.service.loopingcall [None req-03d3ac84-69f7-4407-a09a-9b81ccce1e86 tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 887.062828] env[67752]: DEBUG nova.compute.manager [-] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 887.062828] env[67752]: DEBUG nova.network.neutron [-] [instance: 39479042-b891-4f25-9992-47312c6b6b43] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 887.080613] env[67752]: DEBUG nova.compute.manager [None req-642b4fb0-db2a-4c92-bc8e-8b8564f22880 tempest-ServerPasswordTestJSON-1994449735 tempest-ServerPasswordTestJSON-1994449735-project-member] [instance: 40aaf487-275a-4e75-ac6a-127371e73b2c] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 887.102884] env[67752]: DEBUG oslo_concurrency.lockutils [None req-642b4fb0-db2a-4c92-bc8e-8b8564f22880 tempest-ServerPasswordTestJSON-1994449735 tempest-ServerPasswordTestJSON-1994449735-project-member] Lock "40aaf487-275a-4e75-ac6a-127371e73b2c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 243.826s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 887.115879] env[67752]: DEBUG nova.compute.manager [None req-c9fad532-38a2-4412-bcf7-cfbbd2ea1280 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: d2104921-5ec1-4fb8-8034-52dcdc403a92] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 887.160650] env[67752]: DEBUG nova.compute.manager [None req-c9fad532-38a2-4412-bcf7-cfbbd2ea1280 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] [instance: d2104921-5ec1-4fb8-8034-52dcdc403a92] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 887.184217] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c9fad532-38a2-4412-bcf7-cfbbd2ea1280 tempest-DeleteServersAdminTestJSON-982301001 tempest-DeleteServersAdminTestJSON-982301001-project-member] Lock "d2104921-5ec1-4fb8-8034-52dcdc403a92" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 242.304s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 887.193626] env[67752]: DEBUG nova.compute.manager [None req-fa6ab7e9-8f79-414e-8999-6de8d04f8b75 tempest-AttachInterfacesUnderV243Test-278406362 tempest-AttachInterfacesUnderV243Test-278406362-project-member] [instance: 16f0124d-146a-442a-8738-65f562b928e8] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 887.223721] env[67752]: DEBUG nova.compute.manager [None req-fa6ab7e9-8f79-414e-8999-6de8d04f8b75 tempest-AttachInterfacesUnderV243Test-278406362 tempest-AttachInterfacesUnderV243Test-278406362-project-member] [instance: 16f0124d-146a-442a-8738-65f562b928e8] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 887.243049] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fa6ab7e9-8f79-414e-8999-6de8d04f8b75 tempest-AttachInterfacesUnderV243Test-278406362 tempest-AttachInterfacesUnderV243Test-278406362-project-member] Lock "16f0124d-146a-442a-8738-65f562b928e8" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 240.271s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 887.251958] env[67752]: DEBUG nova.compute.manager [None req-66e56245-c32a-4a6d-8492-e81780b14c1e tempest-ListServerFiltersTestJSON-1646144619 tempest-ListServerFiltersTestJSON-1646144619-project-member] [instance: 8503063e-2888-44c6-bc2a-e9c82be791e2] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 887.275354] env[67752]: DEBUG nova.compute.manager [None req-66e56245-c32a-4a6d-8492-e81780b14c1e tempest-ListServerFiltersTestJSON-1646144619 tempest-ListServerFiltersTestJSON-1646144619-project-member] [instance: 8503063e-2888-44c6-bc2a-e9c82be791e2] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 887.298275] env[67752]: DEBUG oslo_concurrency.lockutils [None req-66e56245-c32a-4a6d-8492-e81780b14c1e tempest-ListServerFiltersTestJSON-1646144619 tempest-ListServerFiltersTestJSON-1646144619-project-member] Lock "8503063e-2888-44c6-bc2a-e9c82be791e2" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 237.003s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 887.307798] env[67752]: DEBUG nova.compute.manager [None req-ec3ed760-d07f-4b6f-b123-835949de2da9 tempest-ListServerFiltersTestJSON-1646144619 tempest-ListServerFiltersTestJSON-1646144619-project-member] [instance: 631002d2-c268-4a6a-9945-c0898b886b93] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 887.327071] env[67752]: DEBUG nova.network.neutron [-] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 887.337041] env[67752]: INFO nova.compute.manager [-] [instance: 39479042-b891-4f25-9992-47312c6b6b43] Took 0.27 seconds to deallocate network for instance. [ 887.346019] env[67752]: DEBUG nova.compute.manager [None req-ec3ed760-d07f-4b6f-b123-835949de2da9 tempest-ListServerFiltersTestJSON-1646144619 tempest-ListServerFiltersTestJSON-1646144619-project-member] [instance: 631002d2-c268-4a6a-9945-c0898b886b93] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 887.367977] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ec3ed760-d07f-4b6f-b123-835949de2da9 tempest-ListServerFiltersTestJSON-1646144619 tempest-ListServerFiltersTestJSON-1646144619-project-member] Lock "631002d2-c268-4a6a-9945-c0898b886b93" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 234.778s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 887.382294] env[67752]: DEBUG nova.compute.manager [None req-162ff140-7e34-4099-9735-3cd37bbd4868 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: 67812e2e-8827-4368-9790-a909c5ebb56b] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 887.408434] env[67752]: DEBUG nova.compute.manager [None req-162ff140-7e34-4099-9735-3cd37bbd4868 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: 67812e2e-8827-4368-9790-a909c5ebb56b] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 887.436918] env[67752]: DEBUG oslo_concurrency.lockutils [None req-162ff140-7e34-4099-9735-3cd37bbd4868 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Lock "67812e2e-8827-4368-9790-a909c5ebb56b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 234.172s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 887.441050] env[67752]: DEBUG oslo_concurrency.lockutils [None req-03d3ac84-69f7-4407-a09a-9b81ccce1e86 tempest-ImagesOneServerTestJSON-350574122 tempest-ImagesOneServerTestJSON-350574122-project-member] Lock "39479042-b891-4f25-9992-47312c6b6b43" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.422s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 887.473607] env[67752]: DEBUG nova.compute.manager [None req-5b072838-12e6-4e37-966a-8e553b120493 tempest-ServerActionsTestJSON-376351690 tempest-ServerActionsTestJSON-376351690-project-member] [instance: 5ed08031-0ef8-4766-951a-37b315c55dc7] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 887.501307] env[67752]: DEBUG nova.compute.manager [None req-5b072838-12e6-4e37-966a-8e553b120493 tempest-ServerActionsTestJSON-376351690 tempest-ServerActionsTestJSON-376351690-project-member] [instance: 5ed08031-0ef8-4766-951a-37b315c55dc7] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 887.530765] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5b072838-12e6-4e37-966a-8e553b120493 tempest-ServerActionsTestJSON-376351690 tempest-ServerActionsTestJSON-376351690-project-member] Lock "5ed08031-0ef8-4766-951a-37b315c55dc7" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 233.435s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 887.542019] env[67752]: DEBUG nova.compute.manager [None req-5ebe9f01-ba51-454e-a9fb-85af4a904115 tempest-ListServerFiltersTestJSON-1646144619 tempest-ListServerFiltersTestJSON-1646144619-project-member] [instance: 436d243a-d6d9-4958-94a8-7b09f5d9fdf3] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 887.572378] env[67752]: DEBUG nova.compute.manager [None req-5ebe9f01-ba51-454e-a9fb-85af4a904115 tempest-ListServerFiltersTestJSON-1646144619 tempest-ListServerFiltersTestJSON-1646144619-project-member] [instance: 436d243a-d6d9-4958-94a8-7b09f5d9fdf3] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 887.594091] env[67752]: DEBUG oslo_concurrency.lockutils [None req-5ebe9f01-ba51-454e-a9fb-85af4a904115 tempest-ListServerFiltersTestJSON-1646144619 tempest-ListServerFiltersTestJSON-1646144619-project-member] Lock "436d243a-d6d9-4958-94a8-7b09f5d9fdf3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 232.548s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 887.603673] env[67752]: DEBUG nova.compute.manager [None req-49fb07d2-4df8-4b3a-99c6-543da3a9498a tempest-ServersWithSpecificFlavorTestJSON-1001786271 tempest-ServersWithSpecificFlavorTestJSON-1001786271-project-member] [instance: f8b03251-6f23-4533-a917-f3fdac87888b] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 887.627530] env[67752]: DEBUG nova.compute.manager [None req-49fb07d2-4df8-4b3a-99c6-543da3a9498a tempest-ServersWithSpecificFlavorTestJSON-1001786271 tempest-ServersWithSpecificFlavorTestJSON-1001786271-project-member] [instance: f8b03251-6f23-4533-a917-f3fdac87888b] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 887.649547] env[67752]: DEBUG oslo_concurrency.lockutils [None req-49fb07d2-4df8-4b3a-99c6-543da3a9498a tempest-ServersWithSpecificFlavorTestJSON-1001786271 tempest-ServersWithSpecificFlavorTestJSON-1001786271-project-member] Lock "f8b03251-6f23-4533-a917-f3fdac87888b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 227.184s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 887.659229] env[67752]: DEBUG nova.compute.manager [None req-c63e4090-88c0-446e-b77e-592b55cc7924 tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] [instance: 58d3b02f-3aac-4beb-a111-44f50c7f1bc8] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 887.683181] env[67752]: DEBUG nova.compute.manager [None req-c63e4090-88c0-446e-b77e-592b55cc7924 tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] [instance: 58d3b02f-3aac-4beb-a111-44f50c7f1bc8] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 887.703794] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c63e4090-88c0-446e-b77e-592b55cc7924 tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] Lock "58d3b02f-3aac-4beb-a111-44f50c7f1bc8" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 223.514s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 887.713017] env[67752]: DEBUG nova.compute.manager [None req-060466ab-7897-4299-ab80-97f57b46db01 tempest-ServersTestFqdnHostnames-644753716 tempest-ServersTestFqdnHostnames-644753716-project-member] [instance: 53eb8b0c-aa50-4d7f-8027-2725c815f134] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 887.738867] env[67752]: DEBUG nova.compute.manager [None req-060466ab-7897-4299-ab80-97f57b46db01 tempest-ServersTestFqdnHostnames-644753716 tempest-ServersTestFqdnHostnames-644753716-project-member] [instance: 53eb8b0c-aa50-4d7f-8027-2725c815f134] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 887.761702] env[67752]: DEBUG oslo_concurrency.lockutils [None req-060466ab-7897-4299-ab80-97f57b46db01 tempest-ServersTestFqdnHostnames-644753716 tempest-ServersTestFqdnHostnames-644753716-project-member] Lock "53eb8b0c-aa50-4d7f-8027-2725c815f134" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 216.164s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 887.773882] env[67752]: DEBUG nova.compute.manager [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 887.834252] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 887.834525] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 887.836091] env[67752]: INFO nova.compute.claims [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 888.227395] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a81bdb18-7b90-4ea4-b14f-26055db3e88b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.236418] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8042919-10eb-467e-a445-a1308c78917f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.265912] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14f72f11-e35c-440f-8848-39436eae0c3f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.273331] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab80c3d0-4ae2-46f8-a3eb-0804222c6447 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.286918] env[67752]: DEBUG nova.compute.provider_tree [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 888.297441] env[67752]: DEBUG nova.scheduler.client.report [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 888.311078] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.476s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 888.311614] env[67752]: DEBUG nova.compute.manager [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 888.352194] env[67752]: DEBUG nova.compute.utils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 888.353797] env[67752]: DEBUG nova.compute.manager [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 888.353797] env[67752]: DEBUG nova.network.neutron [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 888.372574] env[67752]: DEBUG nova.compute.manager [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 888.440526] env[67752]: DEBUG nova.compute.manager [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 888.445889] env[67752]: DEBUG nova.policy [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3271e4e7ed794fcaa2cc63084ae29114', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f697d1fb3ee046c2aacf68791c89de95', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 888.473463] env[67752]: DEBUG nova.virt.hardware [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 888.473931] env[67752]: DEBUG nova.virt.hardware [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 888.474382] env[67752]: DEBUG nova.virt.hardware [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 888.474709] env[67752]: DEBUG nova.virt.hardware [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 888.474976] env[67752]: DEBUG nova.virt.hardware [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 888.475247] env[67752]: DEBUG nova.virt.hardware [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 888.475580] env[67752]: DEBUG nova.virt.hardware [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 888.475969] env[67752]: DEBUG nova.virt.hardware [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 888.476263] env[67752]: DEBUG nova.virt.hardware [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 888.476529] env[67752]: DEBUG nova.virt.hardware [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 888.476800] env[67752]: DEBUG nova.virt.hardware [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 888.479380] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-567f7c82-03cc-40dc-ac7e-9028b4cc9d0e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 888.487846] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a2469886-561a-49f5-bfbf-f2206150cf33 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 889.032059] env[67752]: DEBUG nova.network.neutron [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Successfully created port: 9d0c2174-8b37-4e9b-b7fb-5f6e5d3b9bdf {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 889.231358] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e3a1bc7e-3511-4319-bcee-07db92c8e9a1 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Acquiring lock "411bafb7-ef72-4529-b3f3-0b4955f23788" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 890.513789] env[67752]: DEBUG nova.network.neutron [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Successfully updated port: 9d0c2174-8b37-4e9b-b7fb-5f6e5d3b9bdf {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 890.533403] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Acquiring lock "refresh_cache-411bafb7-ef72-4529-b3f3-0b4955f23788" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 890.534161] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Acquired lock "refresh_cache-411bafb7-ef72-4529-b3f3-0b4955f23788" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 890.536057] env[67752]: DEBUG nova.network.neutron [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 890.565298] env[67752]: DEBUG nova.compute.manager [req-d7ebbbc0-ee5d-4a5e-97a3-279fa2b6c5e5 req-63f442ec-7686-4efc-92d4-0c96c79ecad6 service nova] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Received event network-vif-plugged-9d0c2174-8b37-4e9b-b7fb-5f6e5d3b9bdf {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 890.565605] env[67752]: DEBUG oslo_concurrency.lockutils [req-d7ebbbc0-ee5d-4a5e-97a3-279fa2b6c5e5 req-63f442ec-7686-4efc-92d4-0c96c79ecad6 service nova] Acquiring lock "411bafb7-ef72-4529-b3f3-0b4955f23788-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 890.565843] env[67752]: DEBUG oslo_concurrency.lockutils [req-d7ebbbc0-ee5d-4a5e-97a3-279fa2b6c5e5 req-63f442ec-7686-4efc-92d4-0c96c79ecad6 service nova] Lock "411bafb7-ef72-4529-b3f3-0b4955f23788-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 890.566407] env[67752]: DEBUG oslo_concurrency.lockutils [req-d7ebbbc0-ee5d-4a5e-97a3-279fa2b6c5e5 req-63f442ec-7686-4efc-92d4-0c96c79ecad6 service nova] Lock "411bafb7-ef72-4529-b3f3-0b4955f23788-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 890.566616] env[67752]: DEBUG nova.compute.manager [req-d7ebbbc0-ee5d-4a5e-97a3-279fa2b6c5e5 req-63f442ec-7686-4efc-92d4-0c96c79ecad6 service nova] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] No waiting events found dispatching network-vif-plugged-9d0c2174-8b37-4e9b-b7fb-5f6e5d3b9bdf {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 890.566865] env[67752]: WARNING nova.compute.manager [req-d7ebbbc0-ee5d-4a5e-97a3-279fa2b6c5e5 req-63f442ec-7686-4efc-92d4-0c96c79ecad6 service nova] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Received unexpected event network-vif-plugged-9d0c2174-8b37-4e9b-b7fb-5f6e5d3b9bdf for instance with vm_state building and task_state deleting. [ 890.603446] env[67752]: DEBUG nova.network.neutron [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 890.960631] env[67752]: DEBUG nova.network.neutron [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Updating instance_info_cache with network_info: [{"id": "9d0c2174-8b37-4e9b-b7fb-5f6e5d3b9bdf", "address": "fa:16:3e:66:78:42", "network": {"id": "7f463785-48ad-4f48-974e-6695e96458ea", "bridge": "br-int", "label": "tempest-FloatingIPsAssociationTestJSON-387264553-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "f697d1fb3ee046c2aacf68791c89de95", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "22b8c642-38ad-4c11-9051-145ab3bc54f2", "external-id": "nsx-vlan-transportzone-247", "segmentation_id": 247, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9d0c2174-8b", "ovs_interfaceid": "9d0c2174-8b37-4e9b-b7fb-5f6e5d3b9bdf", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 890.978027] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Releasing lock "refresh_cache-411bafb7-ef72-4529-b3f3-0b4955f23788" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 890.978027] env[67752]: DEBUG nova.compute.manager [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Instance network_info: |[{"id": "9d0c2174-8b37-4e9b-b7fb-5f6e5d3b9bdf", "address": "fa:16:3e:66:78:42", "network": {"id": "7f463785-48ad-4f48-974e-6695e96458ea", "bridge": "br-int", "label": "tempest-FloatingIPsAssociationTestJSON-387264553-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "f697d1fb3ee046c2aacf68791c89de95", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "22b8c642-38ad-4c11-9051-145ab3bc54f2", "external-id": "nsx-vlan-transportzone-247", "segmentation_id": 247, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9d0c2174-8b", "ovs_interfaceid": "9d0c2174-8b37-4e9b-b7fb-5f6e5d3b9bdf", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 890.978329] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:66:78:42', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '22b8c642-38ad-4c11-9051-145ab3bc54f2', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '9d0c2174-8b37-4e9b-b7fb-5f6e5d3b9bdf', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 890.989111] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Creating folder: Project (f697d1fb3ee046c2aacf68791c89de95). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 890.989897] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-61739b42-a5ea-4ecf-8039-49f50d6b26c7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 891.003598] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Created folder: Project (f697d1fb3ee046c2aacf68791c89de95) in parent group-v639722. [ 891.004226] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Creating folder: Instances. Parent ref: group-v639775. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 891.004477] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-1d63dc7c-c070-41a6-967b-b8cc2741821f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 891.015018] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Created folder: Instances in parent group-v639775. [ 891.015385] env[67752]: DEBUG oslo.service.loopingcall [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 891.016036] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 891.016345] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-362e0eaa-9eec-48a0-94cb-d433bf2bbc5b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 891.051571] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 891.051571] env[67752]: value = "task-3199688" [ 891.051571] env[67752]: _type = "Task" [ 891.051571] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 891.066183] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199688, 'name': CreateVM_Task} progress is 6%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 891.563299] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199688, 'name': CreateVM_Task, 'duration_secs': 0.305247} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 891.563570] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 891.564030] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 891.564162] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 891.564487] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 891.564732] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b9ad1487-1736-420c-886e-a07ecbf1d753 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 891.571037] env[67752]: DEBUG oslo_vmware.api [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Waiting for the task: (returnval){ [ 891.571037] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52783872-b7bf-899e-4925-beeeb0d11ba1" [ 891.571037] env[67752]: _type = "Task" [ 891.571037] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 891.577809] env[67752]: DEBUG oslo_vmware.api [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52783872-b7bf-899e-4925-beeeb0d11ba1, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 892.084953] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 892.085238] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 892.085448] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 892.632111] env[67752]: DEBUG nova.compute.manager [req-a504d3c0-9bf6-47fd-aaf7-30827f2ff98a req-64bddbdf-b5c4-4fb6-9023-bd59133f8278 service nova] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Received event network-changed-9d0c2174-8b37-4e9b-b7fb-5f6e5d3b9bdf {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 892.632368] env[67752]: DEBUG nova.compute.manager [req-a504d3c0-9bf6-47fd-aaf7-30827f2ff98a req-64bddbdf-b5c4-4fb6-9023-bd59133f8278 service nova] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Refreshing instance network info cache due to event network-changed-9d0c2174-8b37-4e9b-b7fb-5f6e5d3b9bdf. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 892.632538] env[67752]: DEBUG oslo_concurrency.lockutils [req-a504d3c0-9bf6-47fd-aaf7-30827f2ff98a req-64bddbdf-b5c4-4fb6-9023-bd59133f8278 service nova] Acquiring lock "refresh_cache-411bafb7-ef72-4529-b3f3-0b4955f23788" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 892.632850] env[67752]: DEBUG oslo_concurrency.lockutils [req-a504d3c0-9bf6-47fd-aaf7-30827f2ff98a req-64bddbdf-b5c4-4fb6-9023-bd59133f8278 service nova] Acquired lock "refresh_cache-411bafb7-ef72-4529-b3f3-0b4955f23788" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 892.632850] env[67752]: DEBUG nova.network.neutron [req-a504d3c0-9bf6-47fd-aaf7-30827f2ff98a req-64bddbdf-b5c4-4fb6-9023-bd59133f8278 service nova] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Refreshing network info cache for port 9d0c2174-8b37-4e9b-b7fb-5f6e5d3b9bdf {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 893.177573] env[67752]: DEBUG nova.network.neutron [req-a504d3c0-9bf6-47fd-aaf7-30827f2ff98a req-64bddbdf-b5c4-4fb6-9023-bd59133f8278 service nova] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Updated VIF entry in instance network info cache for port 9d0c2174-8b37-4e9b-b7fb-5f6e5d3b9bdf. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 893.177959] env[67752]: DEBUG nova.network.neutron [req-a504d3c0-9bf6-47fd-aaf7-30827f2ff98a req-64bddbdf-b5c4-4fb6-9023-bd59133f8278 service nova] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Updating instance_info_cache with network_info: [{"id": "9d0c2174-8b37-4e9b-b7fb-5f6e5d3b9bdf", "address": "fa:16:3e:66:78:42", "network": {"id": "7f463785-48ad-4f48-974e-6695e96458ea", "bridge": "br-int", "label": "tempest-FloatingIPsAssociationTestJSON-387264553-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "f697d1fb3ee046c2aacf68791c89de95", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "22b8c642-38ad-4c11-9051-145ab3bc54f2", "external-id": "nsx-vlan-transportzone-247", "segmentation_id": 247, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9d0c2174-8b", "ovs_interfaceid": "9d0c2174-8b37-4e9b-b7fb-5f6e5d3b9bdf", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 893.193388] env[67752]: DEBUG oslo_concurrency.lockutils [req-a504d3c0-9bf6-47fd-aaf7-30827f2ff98a req-64bddbdf-b5c4-4fb6-9023-bd59133f8278 service nova] Releasing lock "refresh_cache-411bafb7-ef72-4529-b3f3-0b4955f23788" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 894.230666] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Acquiring lock "3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 894.231328] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Lock "3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 896.630666] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 896.630970] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 896.656778] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 896.657231] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 897.634940] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 897.635280] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 897.635280] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 897.655533] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 897.655692] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 897.655901] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 897.656081] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 897.656217] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 897.656341] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 897.656461] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 897.656583] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 897.656703] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 897.656844] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 897.656973] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 897.657554] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 897.657771] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 897.657939] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 898.635030] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 898.647081] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 898.647305] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 898.647477] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 898.647632] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 898.648884] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6b03ed10-0183-4511-9a5f-71d9c8b45c63 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 898.657848] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7d1805e9-d12d-4d8a-a7fe-f748fc7f598f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 898.672729] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0930968-1763-4485-865e-39748dfda77c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 898.679197] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c3ab0af3-6f3d-4b9d-ad36-b413319b4c3f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 898.708696] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181005MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 898.708885] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 898.709155] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 898.784619] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 898.784832] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 898.784998] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b0cc1acd-0e33-4547-a43b-549b3de81728 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 898.785174] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance eca7c94f-9bd1-4c15-988e-d4e33086ba40 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 898.785328] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ff4895f8-0374-4b34-a5e8-d3ba24f840a5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 898.785477] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ca18af6b-6b80-44f3-bd8b-f61e2596acd7 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 898.785638] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 02942b29-2343-441e-9e2b-aca2af57c849 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 898.785831] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ebd6c719-32ae-410c-a444-db7ad3f8a9c5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 898.786040] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 898.786170] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 411bafb7-ef72-4529-b3f3-0b4955f23788 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 898.797895] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ada70883-9bef-496e-a65f-ca84f6ee6211 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 898.809225] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 62f05780-7639-4a4f-975f-3f0863118c99 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 898.819807] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 898.829749] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b271cd73-a148-48d6-bed9-6a99512457bd has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 898.839501] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fe967117-5b16-4b5e-aeb7-626e78fee756 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 898.848734] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 07fe1ba8-e5c2-421f-98db-680d90f39913 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 898.857930] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance a3a99c59-0c99-424e-b86c-5f58b494b4ef has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 898.866944] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 505f538c-051f-42e2-9da0-057e29bbb009 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 898.876242] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance a4bd5964-ee9e-44dc-a691-2382dff58c36 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 898.885647] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b800574b-cf7f-458b-ad5c-7c5c3dc5044f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 898.894814] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ee2343dc-52da-4327-afb2-83a774cb6fd0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 898.903090] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 017d212b-25f6-4b00-a1f4-090077a1fe72 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 898.911703] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 15a38322-5e08-4914-b609-a9b5941c9253 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 898.920529] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 0a502eb7-d333-4a4c-a75b-710bf7921871 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 898.929683] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9ad07396-2d8c-4277-b868-206e309631f0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 898.938320] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 898.938564] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 898.938712] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 899.232793] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-725c6416-2569-4931-9ba8-3fc19d775979 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.240536] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c73f34d8-5e21-418d-afda-b2af14980669 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.269348] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-adc2bccd-f1aa-49e3-9bb0-e29451e318e3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.276351] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8f34302-955c-4663-b4f2-c44fe41f8ee9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 899.288934] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 899.299871] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 899.315890] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 899.316114] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.607s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 900.316481] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 900.635039] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 932.037343] env[67752]: WARNING oslo_vmware.rw_handles [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 932.037343] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 932.037343] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 932.037343] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 932.037343] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 932.037343] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 932.037343] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 932.037343] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 932.037343] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 932.037343] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 932.037343] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 932.037343] env[67752]: ERROR oslo_vmware.rw_handles [ 932.038042] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/51d21b0f-797d-4881-989c-664ebcb407f9/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 932.039811] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 932.040988] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Copying Virtual Disk [datastore2] vmware_temp/51d21b0f-797d-4881-989c-664ebcb407f9/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/51d21b0f-797d-4881-989c-664ebcb407f9/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 932.040988] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-b0db2695-4f7b-4cf1-9c23-14269bd5b4d4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 932.050470] env[67752]: DEBUG oslo_vmware.api [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Waiting for the task: (returnval){ [ 932.050470] env[67752]: value = "task-3199689" [ 932.050470] env[67752]: _type = "Task" [ 932.050470] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 932.058207] env[67752]: DEBUG oslo_vmware.api [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Task: {'id': task-3199689, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 932.560944] env[67752]: DEBUG oslo_vmware.exceptions [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 932.561261] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 932.561827] env[67752]: ERROR nova.compute.manager [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 932.561827] env[67752]: Faults: ['InvalidArgument'] [ 932.561827] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Traceback (most recent call last): [ 932.561827] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 932.561827] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] yield resources [ 932.561827] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 932.561827] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] self.driver.spawn(context, instance, image_meta, [ 932.561827] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 932.561827] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] self._vmops.spawn(context, instance, image_meta, injected_files, [ 932.561827] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 932.561827] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] self._fetch_image_if_missing(context, vi) [ 932.561827] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 932.562514] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] image_cache(vi, tmp_image_ds_loc) [ 932.562514] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 932.562514] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] vm_util.copy_virtual_disk( [ 932.562514] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 932.562514] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] session._wait_for_task(vmdk_copy_task) [ 932.562514] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 932.562514] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] return self.wait_for_task(task_ref) [ 932.562514] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 932.562514] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] return evt.wait() [ 932.562514] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 932.562514] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] result = hub.switch() [ 932.562514] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 932.562514] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] return self.greenlet.switch() [ 932.563262] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 932.563262] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] self.f(*self.args, **self.kw) [ 932.563262] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 932.563262] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] raise exceptions.translate_fault(task_info.error) [ 932.563262] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 932.563262] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Faults: ['InvalidArgument'] [ 932.563262] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] [ 932.563262] env[67752]: INFO nova.compute.manager [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Terminating instance [ 932.563755] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 932.563969] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 932.564328] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-ccd7f8f2-e10d-421a-93ee-7480b351d28c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 932.566881] env[67752]: DEBUG nova.compute.manager [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 932.567093] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 932.567812] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-178508ac-f236-48ba-9172-59563ba98318 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 932.574576] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 932.574850] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-c3543896-d2e9-4124-9893-acb660fd38ed {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 932.577106] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 932.577301] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 932.578362] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-f7f7caa6-e8c0-4084-bca8-37490768ac81 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 932.583198] env[67752]: DEBUG oslo_vmware.api [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Waiting for the task: (returnval){ [ 932.583198] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]528fd2ce-d3b3-a8b0-d207-b0ee1aa24a5b" [ 932.583198] env[67752]: _type = "Task" [ 932.583198] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 932.590226] env[67752]: DEBUG oslo_vmware.api [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]528fd2ce-d3b3-a8b0-d207-b0ee1aa24a5b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 932.638025] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 932.638025] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 932.638025] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Deleting the datastore file [datastore2] 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 932.638025] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-2299f5a7-4f7a-4f57-aa38-7107e36dda03 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 932.643592] env[67752]: DEBUG oslo_vmware.api [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Waiting for the task: (returnval){ [ 932.643592] env[67752]: value = "task-3199691" [ 932.643592] env[67752]: _type = "Task" [ 932.643592] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 932.651641] env[67752]: DEBUG oslo_vmware.api [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Task: {'id': task-3199691, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 933.093844] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 933.094142] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Creating directory with path [datastore2] vmware_temp/ad1368fa-267b-4fca-9327-d9e7cb6cb859/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 933.094393] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-94df8ef0-5e38-4ea0-93bb-7aae59b1a68e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.105380] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Created directory with path [datastore2] vmware_temp/ad1368fa-267b-4fca-9327-d9e7cb6cb859/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 933.105594] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Fetch image to [datastore2] vmware_temp/ad1368fa-267b-4fca-9327-d9e7cb6cb859/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 933.105766] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/ad1368fa-267b-4fca-9327-d9e7cb6cb859/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 933.106517] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6be12d94-7c57-4a8c-8e68-3e13c7b7378a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.114055] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fc373a91-9e49-4ab2-828a-7c486cb86d76 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.122875] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ddb446a7-0fe9-4744-9234-c219353b2d04 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.156517] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f95e07b5-a25d-4a68-aac7-8431126def0d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.163243] env[67752]: DEBUG oslo_vmware.api [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Task: {'id': task-3199691, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.075981} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 933.164655] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 933.164856] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 933.165045] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 933.165227] env[67752]: INFO nova.compute.manager [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Took 0.60 seconds to destroy the instance on the hypervisor. [ 933.166982] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-5452669f-0b82-4a16-a092-d6de66fafa7e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.168877] env[67752]: DEBUG nova.compute.claims [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 933.169071] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 933.169314] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 933.191982] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 933.249379] env[67752]: DEBUG oslo_vmware.rw_handles [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/ad1368fa-267b-4fca-9327-d9e7cb6cb859/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 933.310690] env[67752]: DEBUG oslo_vmware.rw_handles [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 933.310883] env[67752]: DEBUG oslo_vmware.rw_handles [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Closing write handle for https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/ad1368fa-267b-4fca-9327-d9e7cb6cb859/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 933.554609] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0287a81-c3ab-4f2d-b5af-072f6d8c20ab {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.562231] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0a742c21-5f02-46c3-b3a1-229d7a605b72 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.591878] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4798180-d8a4-4eca-8516-754fbc483833 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.598954] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56c62513-17db-45ec-a059-be66261f055f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 933.611737] env[67752]: DEBUG nova.compute.provider_tree [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 933.620742] env[67752]: DEBUG nova.scheduler.client.report [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 933.640585] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.470s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 933.640585] env[67752]: ERROR nova.compute.manager [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 933.640585] env[67752]: Faults: ['InvalidArgument'] [ 933.640585] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Traceback (most recent call last): [ 933.640585] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 933.640585] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] self.driver.spawn(context, instance, image_meta, [ 933.640585] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 933.640585] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] self._vmops.spawn(context, instance, image_meta, injected_files, [ 933.640585] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 933.640585] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] self._fetch_image_if_missing(context, vi) [ 933.641023] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 933.641023] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] image_cache(vi, tmp_image_ds_loc) [ 933.641023] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 933.641023] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] vm_util.copy_virtual_disk( [ 933.641023] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 933.641023] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] session._wait_for_task(vmdk_copy_task) [ 933.641023] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 933.641023] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] return self.wait_for_task(task_ref) [ 933.641023] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 933.641023] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] return evt.wait() [ 933.641023] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 933.641023] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] result = hub.switch() [ 933.641023] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 933.641400] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] return self.greenlet.switch() [ 933.641400] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 933.641400] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] self.f(*self.args, **self.kw) [ 933.641400] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 933.641400] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] raise exceptions.translate_fault(task_info.error) [ 933.641400] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 933.641400] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Faults: ['InvalidArgument'] [ 933.641400] env[67752]: ERROR nova.compute.manager [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] [ 933.641400] env[67752]: DEBUG nova.compute.utils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 933.642715] env[67752]: DEBUG nova.compute.manager [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Build of instance 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394 was re-scheduled: A specified parameter was not correct: fileType [ 933.642715] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 933.643104] env[67752]: DEBUG nova.compute.manager [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 933.643373] env[67752]: DEBUG nova.compute.manager [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 933.643583] env[67752]: DEBUG nova.compute.manager [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 933.643774] env[67752]: DEBUG nova.network.neutron [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 934.165643] env[67752]: DEBUG nova.network.neutron [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 934.179908] env[67752]: INFO nova.compute.manager [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Took 0.53 seconds to deallocate network for instance. [ 934.291489] env[67752]: INFO nova.scheduler.client.report [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Deleted allocations for instance 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394 [ 934.316431] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6b8c9489-72ed-4fc1-a408-0a45f02b383b tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 329.802s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 934.317698] env[67752]: DEBUG oslo_concurrency.lockutils [None req-47057ad5-2f30-40e2-a8a5-1711e6b7e520 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 131.182s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 934.317983] env[67752]: DEBUG oslo_concurrency.lockutils [None req-47057ad5-2f30-40e2-a8a5-1711e6b7e520 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquiring lock "9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 934.318228] env[67752]: DEBUG oslo_concurrency.lockutils [None req-47057ad5-2f30-40e2-a8a5-1711e6b7e520 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 934.318404] env[67752]: DEBUG oslo_concurrency.lockutils [None req-47057ad5-2f30-40e2-a8a5-1711e6b7e520 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 934.320457] env[67752]: INFO nova.compute.manager [None req-47057ad5-2f30-40e2-a8a5-1711e6b7e520 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Terminating instance [ 934.322297] env[67752]: DEBUG nova.compute.manager [None req-47057ad5-2f30-40e2-a8a5-1711e6b7e520 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 934.322368] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-47057ad5-2f30-40e2-a8a5-1711e6b7e520 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 934.323125] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f6521da8-82d5-41c5-8c4c-ddc925614502 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 934.333859] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94849d70-411f-4b6e-9bfe-92902dcd2fc9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 934.344169] env[67752]: DEBUG nova.compute.manager [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 934.368344] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-47057ad5-2f30-40e2-a8a5-1711e6b7e520 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394 could not be found. [ 934.368428] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-47057ad5-2f30-40e2-a8a5-1711e6b7e520 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 934.368609] env[67752]: INFO nova.compute.manager [None req-47057ad5-2f30-40e2-a8a5-1711e6b7e520 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Took 0.05 seconds to destroy the instance on the hypervisor. [ 934.368855] env[67752]: DEBUG oslo.service.loopingcall [None req-47057ad5-2f30-40e2-a8a5-1711e6b7e520 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 934.369074] env[67752]: DEBUG nova.compute.manager [-] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 934.369169] env[67752]: DEBUG nova.network.neutron [-] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 934.397279] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 934.397638] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 934.399700] env[67752]: INFO nova.compute.claims [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 934.404760] env[67752]: DEBUG nova.network.neutron [-] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 934.417020] env[67752]: INFO nova.compute.manager [-] [instance: 9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394] Took 0.05 seconds to deallocate network for instance. [ 934.533848] env[67752]: DEBUG oslo_concurrency.lockutils [None req-47057ad5-2f30-40e2-a8a5-1711e6b7e520 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "9fa7b3ec-01ee-4d7c-abb1-73a3b8a78394" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.216s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 934.772056] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-78dfe24e-ede1-43c8-adf2-a5789c3f0cd5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 934.780595] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40adcc57-9904-455c-9dad-fe224fa20007 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 934.812254] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ae947cb-71e5-484f-bc0b-1f4cff2e3cc2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 934.819452] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-661d34da-2ee0-4ae7-a003-0e2126e8880b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 934.833655] env[67752]: DEBUG nova.compute.provider_tree [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 934.842865] env[67752]: DEBUG nova.scheduler.client.report [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 934.859233] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.462s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 934.859725] env[67752]: DEBUG nova.compute.manager [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 934.897718] env[67752]: DEBUG nova.compute.utils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 934.898786] env[67752]: DEBUG nova.compute.manager [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 934.898952] env[67752]: DEBUG nova.network.neutron [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 934.908508] env[67752]: DEBUG nova.compute.manager [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 934.980996] env[67752]: DEBUG nova.compute.manager [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 935.002407] env[67752]: DEBUG nova.virt.hardware [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 935.002655] env[67752]: DEBUG nova.virt.hardware [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 935.002818] env[67752]: DEBUG nova.virt.hardware [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 935.003013] env[67752]: DEBUG nova.virt.hardware [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 935.003173] env[67752]: DEBUG nova.virt.hardware [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 935.003320] env[67752]: DEBUG nova.virt.hardware [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 935.003528] env[67752]: DEBUG nova.virt.hardware [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 935.003687] env[67752]: DEBUG nova.virt.hardware [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 935.003852] env[67752]: DEBUG nova.virt.hardware [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 935.004054] env[67752]: DEBUG nova.virt.hardware [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 935.004251] env[67752]: DEBUG nova.virt.hardware [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 935.005133] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-13e84561-06fd-4f6a-aadb-12f53d8339fd {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 935.013051] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d68a96f1-9873-4a93-a1c1-58097376bb46 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 935.028933] env[67752]: DEBUG nova.policy [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '0afcf1b51c7945b19c8a4cbea27f9102', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5c5fa0e72f934bbf86b335ef35e6074c', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 935.613950] env[67752]: DEBUG nova.network.neutron [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Successfully created port: 2d8f8b37-9afc-46f2-af1a-62fcd82f8c62 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 935.745348] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquiring lock "f6499ee5-cb24-44e1-9fe7-c19feeb732eb" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 935.745823] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "f6499ee5-cb24-44e1-9fe7-c19feeb732eb" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 937.024375] env[67752]: DEBUG nova.network.neutron [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Successfully updated port: 2d8f8b37-9afc-46f2-af1a-62fcd82f8c62 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 937.036665] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Acquiring lock "refresh_cache-ada70883-9bef-496e-a65f-ca84f6ee6211" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 937.037033] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Acquired lock "refresh_cache-ada70883-9bef-496e-a65f-ca84f6ee6211" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 937.037324] env[67752]: DEBUG nova.network.neutron [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 937.103803] env[67752]: DEBUG nova.network.neutron [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 937.180425] env[67752]: DEBUG nova.compute.manager [req-8d11b4ca-1542-4298-b49e-e8bf6da3a624 req-fe3a79f2-1605-4fbf-929d-38535c524347 service nova] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Received event network-vif-plugged-2d8f8b37-9afc-46f2-af1a-62fcd82f8c62 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 937.180663] env[67752]: DEBUG oslo_concurrency.lockutils [req-8d11b4ca-1542-4298-b49e-e8bf6da3a624 req-fe3a79f2-1605-4fbf-929d-38535c524347 service nova] Acquiring lock "ada70883-9bef-496e-a65f-ca84f6ee6211-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 937.180836] env[67752]: DEBUG oslo_concurrency.lockutils [req-8d11b4ca-1542-4298-b49e-e8bf6da3a624 req-fe3a79f2-1605-4fbf-929d-38535c524347 service nova] Lock "ada70883-9bef-496e-a65f-ca84f6ee6211-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 937.181035] env[67752]: DEBUG oslo_concurrency.lockutils [req-8d11b4ca-1542-4298-b49e-e8bf6da3a624 req-fe3a79f2-1605-4fbf-929d-38535c524347 service nova] Lock "ada70883-9bef-496e-a65f-ca84f6ee6211-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 937.181205] env[67752]: DEBUG nova.compute.manager [req-8d11b4ca-1542-4298-b49e-e8bf6da3a624 req-fe3a79f2-1605-4fbf-929d-38535c524347 service nova] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] No waiting events found dispatching network-vif-plugged-2d8f8b37-9afc-46f2-af1a-62fcd82f8c62 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 937.181367] env[67752]: WARNING nova.compute.manager [req-8d11b4ca-1542-4298-b49e-e8bf6da3a624 req-fe3a79f2-1605-4fbf-929d-38535c524347 service nova] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Received unexpected event network-vif-plugged-2d8f8b37-9afc-46f2-af1a-62fcd82f8c62 for instance with vm_state building and task_state spawning. [ 937.413233] env[67752]: DEBUG nova.network.neutron [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Updating instance_info_cache with network_info: [{"id": "2d8f8b37-9afc-46f2-af1a-62fcd82f8c62", "address": "fa:16:3e:f6:d2:85", "network": {"id": "06318e4d-d69d-4aee-b024-8f813d0b700d", "bridge": "br-int", "label": "tempest-ServerMetadataTestJSON-1131465642-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.13", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "5c5fa0e72f934bbf86b335ef35e6074c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "4e2ec358-9bc5-4dd6-8f4e-0d6ec225282a", "external-id": "nsx-vlan-transportzone-843", "segmentation_id": 843, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap2d8f8b37-9a", "ovs_interfaceid": "2d8f8b37-9afc-46f2-af1a-62fcd82f8c62", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 937.427538] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Releasing lock "refresh_cache-ada70883-9bef-496e-a65f-ca84f6ee6211" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 937.427850] env[67752]: DEBUG nova.compute.manager [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Instance network_info: |[{"id": "2d8f8b37-9afc-46f2-af1a-62fcd82f8c62", "address": "fa:16:3e:f6:d2:85", "network": {"id": "06318e4d-d69d-4aee-b024-8f813d0b700d", "bridge": "br-int", "label": "tempest-ServerMetadataTestJSON-1131465642-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.13", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "5c5fa0e72f934bbf86b335ef35e6074c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "4e2ec358-9bc5-4dd6-8f4e-0d6ec225282a", "external-id": "nsx-vlan-transportzone-843", "segmentation_id": 843, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap2d8f8b37-9a", "ovs_interfaceid": "2d8f8b37-9afc-46f2-af1a-62fcd82f8c62", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 937.428339] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:f6:d2:85', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '4e2ec358-9bc5-4dd6-8f4e-0d6ec225282a', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '2d8f8b37-9afc-46f2-af1a-62fcd82f8c62', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 937.436037] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Creating folder: Project (5c5fa0e72f934bbf86b335ef35e6074c). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 937.436422] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-f516eaf2-bb0c-4650-a212-f606e48bf04f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.447262] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Created folder: Project (5c5fa0e72f934bbf86b335ef35e6074c) in parent group-v639722. [ 937.447445] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Creating folder: Instances. Parent ref: group-v639778. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 937.447666] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-2ac151b8-eda3-41e2-a2b7-f8f56806fcb9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.456530] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Created folder: Instances in parent group-v639778. [ 937.456760] env[67752]: DEBUG oslo.service.loopingcall [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 937.457197] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 937.457197] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-3e5eddba-0caa-4bf8-bba3-3490cf679c82 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.475826] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 937.475826] env[67752]: value = "task-3199694" [ 937.475826] env[67752]: _type = "Task" [ 937.475826] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 937.482962] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199694, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 937.985728] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199694, 'name': CreateVM_Task, 'duration_secs': 0.357912} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 937.985728] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 937.986133] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 937.986306] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 937.986644] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 937.986898] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-947ff975-9a67-45fb-a7b6-ed60a3e735bc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 937.991295] env[67752]: DEBUG oslo_vmware.api [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Waiting for the task: (returnval){ [ 937.991295] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52a82c2c-9f41-8a9d-5a8f-f78a055cf7f0" [ 937.991295] env[67752]: _type = "Task" [ 937.991295] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 937.998722] env[67752]: DEBUG oslo_vmware.api [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52a82c2c-9f41-8a9d-5a8f-f78a055cf7f0, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 938.503689] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 938.503689] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 938.503689] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 939.501635] env[67752]: DEBUG nova.compute.manager [req-786a8c7b-e1dd-4495-8104-48993ed86ff3 req-195eb501-73d0-4cc1-964d-127228bb9910 service nova] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Received event network-changed-2d8f8b37-9afc-46f2-af1a-62fcd82f8c62 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 939.501852] env[67752]: DEBUG nova.compute.manager [req-786a8c7b-e1dd-4495-8104-48993ed86ff3 req-195eb501-73d0-4cc1-964d-127228bb9910 service nova] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Refreshing instance network info cache due to event network-changed-2d8f8b37-9afc-46f2-af1a-62fcd82f8c62. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 939.502154] env[67752]: DEBUG oslo_concurrency.lockutils [req-786a8c7b-e1dd-4495-8104-48993ed86ff3 req-195eb501-73d0-4cc1-964d-127228bb9910 service nova] Acquiring lock "refresh_cache-ada70883-9bef-496e-a65f-ca84f6ee6211" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 939.502468] env[67752]: DEBUG oslo_concurrency.lockutils [req-786a8c7b-e1dd-4495-8104-48993ed86ff3 req-195eb501-73d0-4cc1-964d-127228bb9910 service nova] Acquired lock "refresh_cache-ada70883-9bef-496e-a65f-ca84f6ee6211" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 939.502653] env[67752]: DEBUG nova.network.neutron [req-786a8c7b-e1dd-4495-8104-48993ed86ff3 req-195eb501-73d0-4cc1-964d-127228bb9910 service nova] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Refreshing network info cache for port 2d8f8b37-9afc-46f2-af1a-62fcd82f8c62 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 939.573905] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a0281c8f-05df-47c1-ae92-ab4685cca0aa tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Acquiring lock "ada70883-9bef-496e-a65f-ca84f6ee6211" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 939.951985] env[67752]: DEBUG nova.network.neutron [req-786a8c7b-e1dd-4495-8104-48993ed86ff3 req-195eb501-73d0-4cc1-964d-127228bb9910 service nova] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Updated VIF entry in instance network info cache for port 2d8f8b37-9afc-46f2-af1a-62fcd82f8c62. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 939.952514] env[67752]: DEBUG nova.network.neutron [req-786a8c7b-e1dd-4495-8104-48993ed86ff3 req-195eb501-73d0-4cc1-964d-127228bb9910 service nova] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Updating instance_info_cache with network_info: [{"id": "2d8f8b37-9afc-46f2-af1a-62fcd82f8c62", "address": "fa:16:3e:f6:d2:85", "network": {"id": "06318e4d-d69d-4aee-b024-8f813d0b700d", "bridge": "br-int", "label": "tempest-ServerMetadataTestJSON-1131465642-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.13", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "5c5fa0e72f934bbf86b335ef35e6074c", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "4e2ec358-9bc5-4dd6-8f4e-0d6ec225282a", "external-id": "nsx-vlan-transportzone-843", "segmentation_id": 843, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap2d8f8b37-9a", "ovs_interfaceid": "2d8f8b37-9afc-46f2-af1a-62fcd82f8c62", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 939.961669] env[67752]: DEBUG oslo_concurrency.lockutils [req-786a8c7b-e1dd-4495-8104-48993ed86ff3 req-195eb501-73d0-4cc1-964d-127228bb9910 service nova] Releasing lock "refresh_cache-ada70883-9bef-496e-a65f-ca84f6ee6211" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 956.631285] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 957.634627] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 957.634989] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 957.634989] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 957.660075] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 957.660075] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 957.660075] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 957.660075] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 957.660075] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 957.660341] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 957.660341] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 957.660341] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 957.660341] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 957.660341] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 957.660525] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 957.660525] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 957.660525] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 958.634591] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 959.634585] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 959.634924] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 959.634964] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 960.635203] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 960.635623] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 960.648127] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 960.648381] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 960.648595] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 960.648790] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 960.650151] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1c320e6f-8220-4ce9-bbe6-c31a17e22a23 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 960.659249] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1a888b7-35cc-4a22-a71b-e63a64e290fa {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 960.673018] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7446369e-957e-4916-b114-9e251090d2be {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 960.679154] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5fb6392f-c139-4940-a73f-c8c507b734c3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 960.708050] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181001MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 960.708209] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 960.708464] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 960.783533] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 960.783702] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b0cc1acd-0e33-4547-a43b-549b3de81728 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 960.783832] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance eca7c94f-9bd1-4c15-988e-d4e33086ba40 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 960.783954] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ff4895f8-0374-4b34-a5e8-d3ba24f840a5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 960.784086] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ca18af6b-6b80-44f3-bd8b-f61e2596acd7 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 960.784206] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 02942b29-2343-441e-9e2b-aca2af57c849 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 960.784322] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ebd6c719-32ae-410c-a444-db7ad3f8a9c5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 960.784435] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 960.784549] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 411bafb7-ef72-4529-b3f3-0b4955f23788 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 960.784663] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ada70883-9bef-496e-a65f-ca84f6ee6211 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 960.796604] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 62f05780-7639-4a4f-975f-3f0863118c99 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 960.808170] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 960.821148] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b271cd73-a148-48d6-bed9-6a99512457bd has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 960.831171] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fe967117-5b16-4b5e-aeb7-626e78fee756 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 960.841695] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 07fe1ba8-e5c2-421f-98db-680d90f39913 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 960.851713] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance a3a99c59-0c99-424e-b86c-5f58b494b4ef has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 960.860164] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 505f538c-051f-42e2-9da0-057e29bbb009 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 960.870995] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance a4bd5964-ee9e-44dc-a691-2382dff58c36 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 960.884634] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b800574b-cf7f-458b-ad5c-7c5c3dc5044f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 960.895344] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ee2343dc-52da-4327-afb2-83a774cb6fd0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 960.907157] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 017d212b-25f6-4b00-a1f4-090077a1fe72 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 960.916607] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 15a38322-5e08-4914-b609-a9b5941c9253 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 960.925635] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 0a502eb7-d333-4a4c-a75b-710bf7921871 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 960.939599] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9ad07396-2d8c-4277-b868-206e309631f0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 960.949261] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 960.958155] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f6499ee5-cb24-44e1-9fe7-c19feeb732eb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 960.958606] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 960.958886] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 961.245853] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2096796f-44a6-4bdc-9807-8371eaf2e9fb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 961.254112] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3876b1eb-1a4a-4e43-8f3b-c6f011c22b53 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 961.283825] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-539bae99-cfcb-4430-9936-673dc1e2aba0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 961.290787] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e788f1f0-f531-4070-b01b-1535ab39b415 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 961.304067] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 961.313264] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 961.333154] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 961.333362] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.625s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 978.508296] env[67752]: WARNING oslo_vmware.rw_handles [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 978.508296] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 978.508296] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 978.508296] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 978.508296] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 978.508296] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 978.508296] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 978.508296] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 978.508296] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 978.508296] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 978.508296] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 978.508296] env[67752]: ERROR oslo_vmware.rw_handles [ 978.508972] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/ad1368fa-267b-4fca-9327-d9e7cb6cb859/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 978.510936] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 978.511190] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Copying Virtual Disk [datastore2] vmware_temp/ad1368fa-267b-4fca-9327-d9e7cb6cb859/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/ad1368fa-267b-4fca-9327-d9e7cb6cb859/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 978.511486] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-32e6874a-29ee-4e04-beab-98b0d6bffb21 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 978.518890] env[67752]: DEBUG oslo_vmware.api [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Waiting for the task: (returnval){ [ 978.518890] env[67752]: value = "task-3199695" [ 978.518890] env[67752]: _type = "Task" [ 978.518890] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 978.528308] env[67752]: DEBUG oslo_vmware.api [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Task: {'id': task-3199695, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 979.029673] env[67752]: DEBUG oslo_vmware.exceptions [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 979.029992] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 979.030617] env[67752]: ERROR nova.compute.manager [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 979.030617] env[67752]: Faults: ['InvalidArgument'] [ 979.030617] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Traceback (most recent call last): [ 979.030617] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 979.030617] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] yield resources [ 979.030617] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 979.030617] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] self.driver.spawn(context, instance, image_meta, [ 979.030617] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 979.030617] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] self._vmops.spawn(context, instance, image_meta, injected_files, [ 979.030617] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 979.030617] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] self._fetch_image_if_missing(context, vi) [ 979.030617] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 979.030931] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] image_cache(vi, tmp_image_ds_loc) [ 979.030931] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 979.030931] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] vm_util.copy_virtual_disk( [ 979.030931] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 979.030931] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] session._wait_for_task(vmdk_copy_task) [ 979.030931] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 979.030931] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] return self.wait_for_task(task_ref) [ 979.030931] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 979.030931] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] return evt.wait() [ 979.030931] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 979.030931] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] result = hub.switch() [ 979.030931] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 979.030931] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] return self.greenlet.switch() [ 979.031212] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 979.031212] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] self.f(*self.args, **self.kw) [ 979.031212] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 979.031212] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] raise exceptions.translate_fault(task_info.error) [ 979.031212] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 979.031212] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Faults: ['InvalidArgument'] [ 979.031212] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] [ 979.031212] env[67752]: INFO nova.compute.manager [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Terminating instance [ 979.032654] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 979.032894] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 979.033188] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-633ad122-f366-407d-8211-9563e96ebdb0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 979.035595] env[67752]: DEBUG nova.compute.manager [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 979.035817] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 979.036575] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8115433-8607-4c75-97e0-0ce320043286 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 979.043402] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 979.043619] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-ac8fb3c7-7d50-4077-a40f-aa02e8b8db6a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 979.045793] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 979.045966] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 979.046895] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-211514d8-9eb4-4ec4-89e6-d8ee44a2b877 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 979.051718] env[67752]: DEBUG oslo_vmware.api [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Waiting for the task: (returnval){ [ 979.051718] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52ab7320-b4f4-202e-39e3-e1816c0226b4" [ 979.051718] env[67752]: _type = "Task" [ 979.051718] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 979.059085] env[67752]: DEBUG oslo_vmware.api [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52ab7320-b4f4-202e-39e3-e1816c0226b4, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 979.113310] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 979.113556] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 979.113741] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Deleting the datastore file [datastore2] 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 979.114247] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-001644d5-db65-4a96-bca8-be4e7bfcfef0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 979.120355] env[67752]: DEBUG oslo_vmware.api [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Waiting for the task: (returnval){ [ 979.120355] env[67752]: value = "task-3199697" [ 979.120355] env[67752]: _type = "Task" [ 979.120355] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 979.128610] env[67752]: DEBUG oslo_vmware.api [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Task: {'id': task-3199697, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 979.315701] env[67752]: DEBUG oslo_concurrency.lockutils [None req-730dd685-a504-4a15-b85b-9b4395944e0f tempest-ServersListShow296Test-1143284993 tempest-ServersListShow296Test-1143284993-project-member] Acquiring lock "e7d86d5f-f92d-4891-b490-8d0fe37537fb" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 979.315930] env[67752]: DEBUG oslo_concurrency.lockutils [None req-730dd685-a504-4a15-b85b-9b4395944e0f tempest-ServersListShow296Test-1143284993 tempest-ServersListShow296Test-1143284993-project-member] Lock "e7d86d5f-f92d-4891-b490-8d0fe37537fb" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 979.561473] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 979.561765] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Creating directory with path [datastore2] vmware_temp/dac8e05e-4811-4029-a3a4-c5b400186b03/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 979.562086] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-ff1f4817-9f2f-49ae-8ab3-3cbcd3fb0529 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 979.573677] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Created directory with path [datastore2] vmware_temp/dac8e05e-4811-4029-a3a4-c5b400186b03/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 979.573880] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Fetch image to [datastore2] vmware_temp/dac8e05e-4811-4029-a3a4-c5b400186b03/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 979.574059] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/dac8e05e-4811-4029-a3a4-c5b400186b03/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 979.574794] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ff8f9cc0-d9db-44ea-838b-3fd14090b39d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 979.581800] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b948000-2d9d-4cd3-8d3c-7be8f031db34 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 979.591464] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44347b5d-ec92-4ed4-885a-2aae1560834d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 979.625766] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c435822-42b2-49c0-96e9-0c57241b23d0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 979.632813] env[67752]: DEBUG oslo_vmware.api [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Task: {'id': task-3199697, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.073013} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 979.634279] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 979.634491] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 979.634669] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 979.634875] env[67752]: INFO nova.compute.manager [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Took 0.60 seconds to destroy the instance on the hypervisor. [ 979.636680] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-eb306295-8e50-4c8d-a5b1-4ecd0b33c15e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 979.639049] env[67752]: DEBUG nova.compute.claims [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 979.639225] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 979.639439] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 979.661045] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 979.718574] env[67752]: DEBUG oslo_vmware.rw_handles [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/dac8e05e-4811-4029-a3a4-c5b400186b03/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 979.779063] env[67752]: DEBUG oslo_vmware.rw_handles [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 979.779284] env[67752]: DEBUG oslo_vmware.rw_handles [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Closing write handle for https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/dac8e05e-4811-4029-a3a4-c5b400186b03/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 980.031205] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5c9daea5-e1dc-4c7b-a330-2675106fec63 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 980.038348] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-39675475-5bdd-4d9e-9e0c-26c1bcdbe95f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 980.068466] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dad31363-d171-40b6-9710-1f1321e9bcd9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 980.075250] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42b754f0-1da5-42d3-b094-150a9ad7f4dc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 980.087562] env[67752]: DEBUG nova.compute.provider_tree [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 980.096027] env[67752]: DEBUG nova.scheduler.client.report [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 980.109682] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.470s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 980.110207] env[67752]: ERROR nova.compute.manager [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 980.110207] env[67752]: Faults: ['InvalidArgument'] [ 980.110207] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Traceback (most recent call last): [ 980.110207] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 980.110207] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] self.driver.spawn(context, instance, image_meta, [ 980.110207] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 980.110207] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] self._vmops.spawn(context, instance, image_meta, injected_files, [ 980.110207] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 980.110207] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] self._fetch_image_if_missing(context, vi) [ 980.110207] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 980.110207] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] image_cache(vi, tmp_image_ds_loc) [ 980.110207] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 980.110483] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] vm_util.copy_virtual_disk( [ 980.110483] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 980.110483] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] session._wait_for_task(vmdk_copy_task) [ 980.110483] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 980.110483] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] return self.wait_for_task(task_ref) [ 980.110483] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 980.110483] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] return evt.wait() [ 980.110483] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 980.110483] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] result = hub.switch() [ 980.110483] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 980.110483] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] return self.greenlet.switch() [ 980.110483] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 980.110483] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] self.f(*self.args, **self.kw) [ 980.110753] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 980.110753] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] raise exceptions.translate_fault(task_info.error) [ 980.110753] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 980.110753] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Faults: ['InvalidArgument'] [ 980.110753] env[67752]: ERROR nova.compute.manager [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] [ 980.110962] env[67752]: DEBUG nova.compute.utils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 980.112632] env[67752]: DEBUG nova.compute.manager [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Build of instance 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175 was re-scheduled: A specified parameter was not correct: fileType [ 980.112632] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 980.113015] env[67752]: DEBUG nova.compute.manager [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 980.113198] env[67752]: DEBUG nova.compute.manager [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 980.113363] env[67752]: DEBUG nova.compute.manager [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 980.113526] env[67752]: DEBUG nova.network.neutron [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 980.592828] env[67752]: DEBUG nova.network.neutron [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 980.608077] env[67752]: INFO nova.compute.manager [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Took 0.49 seconds to deallocate network for instance. [ 980.750267] env[67752]: INFO nova.scheduler.client.report [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Deleted allocations for instance 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175 [ 980.779632] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a6dd2140-2824-4aef-abe8-cec8075e881a tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 375.477s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 980.781775] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e6527cba-2f5b-4125-8fd0-2ffac9f9cafa tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 176.375s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 980.781775] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e6527cba-2f5b-4125-8fd0-2ffac9f9cafa tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquiring lock "5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 980.781775] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e6527cba-2f5b-4125-8fd0-2ffac9f9cafa tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 980.781775] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e6527cba-2f5b-4125-8fd0-2ffac9f9cafa tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 980.784034] env[67752]: INFO nova.compute.manager [None req-e6527cba-2f5b-4125-8fd0-2ffac9f9cafa tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Terminating instance [ 980.786252] env[67752]: DEBUG nova.compute.manager [None req-e6527cba-2f5b-4125-8fd0-2ffac9f9cafa tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 980.786508] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-e6527cba-2f5b-4125-8fd0-2ffac9f9cafa tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 980.787578] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-928ec7c9-dd55-4709-ab5e-e8128de22428 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 980.797183] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8dd80afc-0d75-4702-aa2b-10df0599b2ce {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 980.809865] env[67752]: DEBUG nova.compute.manager [None req-cb3b4342-144b-4ab6-9200-e49e2d28607f tempest-ServerRescueTestJSONUnderV235-1587357604 tempest-ServerRescueTestJSONUnderV235-1587357604-project-member] [instance: 62f05780-7639-4a4f-975f-3f0863118c99] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 980.839319] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-e6527cba-2f5b-4125-8fd0-2ffac9f9cafa tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175 could not be found. [ 980.839319] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-e6527cba-2f5b-4125-8fd0-2ffac9f9cafa tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 980.839396] env[67752]: INFO nova.compute.manager [None req-e6527cba-2f5b-4125-8fd0-2ffac9f9cafa tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Took 0.05 seconds to destroy the instance on the hypervisor. [ 980.839602] env[67752]: DEBUG oslo.service.loopingcall [None req-e6527cba-2f5b-4125-8fd0-2ffac9f9cafa tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 980.839832] env[67752]: DEBUG nova.compute.manager [-] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 980.839930] env[67752]: DEBUG nova.network.neutron [-] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 980.847400] env[67752]: DEBUG nova.compute.manager [None req-cb3b4342-144b-4ab6-9200-e49e2d28607f tempest-ServerRescueTestJSONUnderV235-1587357604 tempest-ServerRescueTestJSONUnderV235-1587357604-project-member] [instance: 62f05780-7639-4a4f-975f-3f0863118c99] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 980.874887] env[67752]: DEBUG nova.network.neutron [-] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 980.881945] env[67752]: DEBUG oslo_concurrency.lockutils [None req-cb3b4342-144b-4ab6-9200-e49e2d28607f tempest-ServerRescueTestJSONUnderV235-1587357604 tempest-ServerRescueTestJSONUnderV235-1587357604-project-member] Lock "62f05780-7639-4a4f-975f-3f0863118c99" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 201.702s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 980.890361] env[67752]: INFO nova.compute.manager [-] [instance: 5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175] Took 0.05 seconds to deallocate network for instance. [ 980.896158] env[67752]: DEBUG nova.compute.manager [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 980.952933] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 980.953402] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 980.955396] env[67752]: INFO nova.compute.claims [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 981.000844] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e6527cba-2f5b-4125-8fd0-2ffac9f9cafa tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "5d2fb7f8-e4a4-4a9d-be93-6d1b7f30b175" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.220s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 981.358058] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-582c78ad-1128-4c7f-91e0-091854c6205a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 981.368996] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94cf5953-5041-4062-8232-f1855661203c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 981.423716] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-235c5b23-23f7-46af-a6ea-a00b0e6989ef {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 981.434915] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b7a97b7f-8e8d-4485-a7d1-1433834a7ddd {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 981.456648] env[67752]: DEBUG nova.compute.provider_tree [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 981.468993] env[67752]: DEBUG nova.scheduler.client.report [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 981.484315] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.531s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 981.484812] env[67752]: DEBUG nova.compute.manager [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 981.537117] env[67752]: DEBUG nova.compute.utils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 981.537941] env[67752]: DEBUG nova.compute.manager [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 981.538366] env[67752]: DEBUG nova.network.neutron [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 981.547763] env[67752]: DEBUG nova.compute.manager [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 981.636023] env[67752]: DEBUG nova.compute.manager [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 981.643138] env[67752]: DEBUG nova.policy [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '0c384ef517ad4d8b9afb2a92d8101bf6', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '9dd47726f9eb4844be3fc4a892aede82', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 981.666356] env[67752]: DEBUG nova.virt.hardware [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 981.666356] env[67752]: DEBUG nova.virt.hardware [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 981.666356] env[67752]: DEBUG nova.virt.hardware [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 981.666529] env[67752]: DEBUG nova.virt.hardware [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 981.666529] env[67752]: DEBUG nova.virt.hardware [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 981.666832] env[67752]: DEBUG nova.virt.hardware [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 981.667317] env[67752]: DEBUG nova.virt.hardware [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 981.667656] env[67752]: DEBUG nova.virt.hardware [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 981.670017] env[67752]: DEBUG nova.virt.hardware [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 981.670017] env[67752]: DEBUG nova.virt.hardware [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 981.670017] env[67752]: DEBUG nova.virt.hardware [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 981.670017] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ced86220-76f2-4268-9963-8e7dceebe307 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 981.683028] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb105f96-1f68-45f2-8b49-48f100f0e91d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 982.250673] env[67752]: DEBUG nova.network.neutron [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Successfully created port: 137fdc34-b926-456f-a047-d8445f99a612 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 983.412102] env[67752]: DEBUG nova.network.neutron [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Successfully updated port: 137fdc34-b926-456f-a047-d8445f99a612 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 983.426579] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Acquiring lock "refresh_cache-1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 983.426728] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Acquired lock "refresh_cache-1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 983.426876] env[67752]: DEBUG nova.network.neutron [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 983.454962] env[67752]: DEBUG nova.compute.manager [req-cffcc68c-461d-4231-ba9b-2294b2d10a8c req-d6f69f8a-839d-473f-af6e-9df618ac200e service nova] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Received event network-vif-plugged-137fdc34-b926-456f-a047-d8445f99a612 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 983.454962] env[67752]: DEBUG oslo_concurrency.lockutils [req-cffcc68c-461d-4231-ba9b-2294b2d10a8c req-d6f69f8a-839d-473f-af6e-9df618ac200e service nova] Acquiring lock "1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 983.454962] env[67752]: DEBUG oslo_concurrency.lockutils [req-cffcc68c-461d-4231-ba9b-2294b2d10a8c req-d6f69f8a-839d-473f-af6e-9df618ac200e service nova] Lock "1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 983.454962] env[67752]: DEBUG oslo_concurrency.lockutils [req-cffcc68c-461d-4231-ba9b-2294b2d10a8c req-d6f69f8a-839d-473f-af6e-9df618ac200e service nova] Lock "1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 983.455146] env[67752]: DEBUG nova.compute.manager [req-cffcc68c-461d-4231-ba9b-2294b2d10a8c req-d6f69f8a-839d-473f-af6e-9df618ac200e service nova] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] No waiting events found dispatching network-vif-plugged-137fdc34-b926-456f-a047-d8445f99a612 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 983.455146] env[67752]: WARNING nova.compute.manager [req-cffcc68c-461d-4231-ba9b-2294b2d10a8c req-d6f69f8a-839d-473f-af6e-9df618ac200e service nova] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Received unexpected event network-vif-plugged-137fdc34-b926-456f-a047-d8445f99a612 for instance with vm_state building and task_state spawning. [ 983.517775] env[67752]: DEBUG nova.network.neutron [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 983.676626] env[67752]: DEBUG oslo_concurrency.lockutils [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquiring lock "c7081273-b937-4b9f-9369-79162ca200d1" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 983.676864] env[67752]: DEBUG oslo_concurrency.lockutils [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "c7081273-b937-4b9f-9369-79162ca200d1" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 984.031035] env[67752]: DEBUG nova.network.neutron [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Updating instance_info_cache with network_info: [{"id": "137fdc34-b926-456f-a047-d8445f99a612", "address": "fa:16:3e:d9:4e:05", "network": {"id": "eb402755-7872-4bfb-9f9d-289149082210", "bridge": "br-int", "label": "tempest-ServerAddressesTestJSON-801852258-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "9dd47726f9eb4844be3fc4a892aede82", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "8ee1c76d-1a61-4546-85cb-d4bd3c1b35ef", "external-id": "nsx-vlan-transportzone-161", "segmentation_id": 161, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap137fdc34-b9", "ovs_interfaceid": "137fdc34-b926-456f-a047-d8445f99a612", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 984.045236] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Releasing lock "refresh_cache-1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 984.045542] env[67752]: DEBUG nova.compute.manager [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Instance network_info: |[{"id": "137fdc34-b926-456f-a047-d8445f99a612", "address": "fa:16:3e:d9:4e:05", "network": {"id": "eb402755-7872-4bfb-9f9d-289149082210", "bridge": "br-int", "label": "tempest-ServerAddressesTestJSON-801852258-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "9dd47726f9eb4844be3fc4a892aede82", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "8ee1c76d-1a61-4546-85cb-d4bd3c1b35ef", "external-id": "nsx-vlan-transportzone-161", "segmentation_id": 161, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap137fdc34-b9", "ovs_interfaceid": "137fdc34-b926-456f-a047-d8445f99a612", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 984.045957] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:d9:4e:05', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '8ee1c76d-1a61-4546-85cb-d4bd3c1b35ef', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '137fdc34-b926-456f-a047-d8445f99a612', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 984.053620] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Creating folder: Project (9dd47726f9eb4844be3fc4a892aede82). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 984.054181] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-a22f2305-71fa-43fb-8b0e-c99f7655e2d9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 984.063850] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Created folder: Project (9dd47726f9eb4844be3fc4a892aede82) in parent group-v639722. [ 984.064048] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Creating folder: Instances. Parent ref: group-v639781. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 984.065180] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-4cadc58e-ec9e-4f79-be87-8662060cf06d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 984.072263] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Created folder: Instances in parent group-v639781. [ 984.072490] env[67752]: DEBUG oslo.service.loopingcall [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 984.072715] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 984.072934] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-feab7f84-93e4-4dd5-aab0-5ead3d3b13dc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 984.090725] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 984.090725] env[67752]: value = "task-3199700" [ 984.090725] env[67752]: _type = "Task" [ 984.090725] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 984.099139] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199700, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 984.603549] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199700, 'name': CreateVM_Task, 'duration_secs': 0.322972} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 984.603800] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 984.604698] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 984.604952] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 984.605502] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 984.605866] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-5eaedd63-dd25-49d1-9c5a-0ab256add16b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 984.611235] env[67752]: DEBUG oslo_vmware.api [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Waiting for the task: (returnval){ [ 984.611235] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]524e9737-c28d-90d6-9aa9-5ef9d1988093" [ 984.611235] env[67752]: _type = "Task" [ 984.611235] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 984.622807] env[67752]: DEBUG oslo_vmware.api [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]524e9737-c28d-90d6-9aa9-5ef9d1988093, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 985.122677] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 985.122969] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 985.123177] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 985.563856] env[67752]: DEBUG nova.compute.manager [req-9feacdf1-ae1b-4dab-9502-15178f2826a9 req-e1d399cf-e590-4644-a1ae-75f68786543c service nova] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Received event network-changed-137fdc34-b926-456f-a047-d8445f99a612 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 985.563978] env[67752]: DEBUG nova.compute.manager [req-9feacdf1-ae1b-4dab-9502-15178f2826a9 req-e1d399cf-e590-4644-a1ae-75f68786543c service nova] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Refreshing instance network info cache due to event network-changed-137fdc34-b926-456f-a047-d8445f99a612. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 985.564203] env[67752]: DEBUG oslo_concurrency.lockutils [req-9feacdf1-ae1b-4dab-9502-15178f2826a9 req-e1d399cf-e590-4644-a1ae-75f68786543c service nova] Acquiring lock "refresh_cache-1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 985.564364] env[67752]: DEBUG oslo_concurrency.lockutils [req-9feacdf1-ae1b-4dab-9502-15178f2826a9 req-e1d399cf-e590-4644-a1ae-75f68786543c service nova] Acquired lock "refresh_cache-1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 985.564506] env[67752]: DEBUG nova.network.neutron [req-9feacdf1-ae1b-4dab-9502-15178f2826a9 req-e1d399cf-e590-4644-a1ae-75f68786543c service nova] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Refreshing network info cache for port 137fdc34-b926-456f-a047-d8445f99a612 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 986.065716] env[67752]: DEBUG nova.network.neutron [req-9feacdf1-ae1b-4dab-9502-15178f2826a9 req-e1d399cf-e590-4644-a1ae-75f68786543c service nova] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Updated VIF entry in instance network info cache for port 137fdc34-b926-456f-a047-d8445f99a612. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 986.066223] env[67752]: DEBUG nova.network.neutron [req-9feacdf1-ae1b-4dab-9502-15178f2826a9 req-e1d399cf-e590-4644-a1ae-75f68786543c service nova] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Updating instance_info_cache with network_info: [{"id": "137fdc34-b926-456f-a047-d8445f99a612", "address": "fa:16:3e:d9:4e:05", "network": {"id": "eb402755-7872-4bfb-9f9d-289149082210", "bridge": "br-int", "label": "tempest-ServerAddressesTestJSON-801852258-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "9dd47726f9eb4844be3fc4a892aede82", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "8ee1c76d-1a61-4546-85cb-d4bd3c1b35ef", "external-id": "nsx-vlan-transportzone-161", "segmentation_id": 161, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap137fdc34-b9", "ovs_interfaceid": "137fdc34-b926-456f-a047-d8445f99a612", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 986.075619] env[67752]: DEBUG oslo_concurrency.lockutils [req-9feacdf1-ae1b-4dab-9502-15178f2826a9 req-e1d399cf-e590-4644-a1ae-75f68786543c service nova] Releasing lock "refresh_cache-1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 993.515655] env[67752]: DEBUG oslo_concurrency.lockutils [None req-41977d13-8ea4-4f2f-ab4f-655b474eb19b tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Acquiring lock "1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1006.131256] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9ee3a5c0-b3bd-4f15-9ac5-e4a45b38e7a6 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquiring lock "b0cc1acd-0e33-4547-a43b-549b3de81728" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1013.025062] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Acquiring lock "d3940c29-852a-427b-9027-aa4080150724" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1013.025375] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Lock "d3940c29-852a-427b-9027-aa4080150724" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1017.406631] env[67752]: DEBUG oslo_concurrency.lockutils [None req-db8f6f02-176a-4689-a251-166bd122b16f tempest-ListServersNegativeTestJSON-684094107 tempest-ListServersNegativeTestJSON-684094107-project-member] Acquiring lock "9065b97a-05a3-4b62-8f4a-418d5b0b209a" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1017.407570] env[67752]: DEBUG oslo_concurrency.lockutils [None req-db8f6f02-176a-4689-a251-166bd122b16f tempest-ListServersNegativeTestJSON-684094107 tempest-ListServersNegativeTestJSON-684094107-project-member] Lock "9065b97a-05a3-4b62-8f4a-418d5b0b209a" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1017.478797] env[67752]: DEBUG oslo_concurrency.lockutils [None req-db8f6f02-176a-4689-a251-166bd122b16f tempest-ListServersNegativeTestJSON-684094107 tempest-ListServersNegativeTestJSON-684094107-project-member] Acquiring lock "e2fc8dbe-6458-4f5a-9aef-c44ea4d61b08" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1017.479077] env[67752]: DEBUG oslo_concurrency.lockutils [None req-db8f6f02-176a-4689-a251-166bd122b16f tempest-ListServersNegativeTestJSON-684094107 tempest-ListServersNegativeTestJSON-684094107-project-member] Lock "e2fc8dbe-6458-4f5a-9aef-c44ea4d61b08" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1017.518575] env[67752]: DEBUG oslo_concurrency.lockutils [None req-db8f6f02-176a-4689-a251-166bd122b16f tempest-ListServersNegativeTestJSON-684094107 tempest-ListServersNegativeTestJSON-684094107-project-member] Acquiring lock "6da6a17c-b133-4d3c-8b96-6317760a7b3b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1017.518836] env[67752]: DEBUG oslo_concurrency.lockutils [None req-db8f6f02-176a-4689-a251-166bd122b16f tempest-ListServersNegativeTestJSON-684094107 tempest-ListServersNegativeTestJSON-684094107-project-member] Lock "6da6a17c-b133-4d3c-8b96-6317760a7b3b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1018.329375] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1018.329436] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1018.329603] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1018.630176] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1018.750258] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6f4d6066-9124-472e-9aa7-1f9ed12c4f85 tempest-TenantUsagesTestJSON-1814372835 tempest-TenantUsagesTestJSON-1814372835-project-member] Acquiring lock "9f409285-631b-48f9-8b80-a3179c09a237" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1018.750258] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6f4d6066-9124-472e-9aa7-1f9ed12c4f85 tempest-TenantUsagesTestJSON-1814372835 tempest-TenantUsagesTestJSON-1814372835-project-member] Lock "9f409285-631b-48f9-8b80-a3179c09a237" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1019.635643] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1019.635643] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 1019.635643] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 1019.674678] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1019.675129] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1019.675396] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1019.675676] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1019.676159] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1019.676904] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1019.676904] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1019.676904] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1019.676904] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1019.678464] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1019.678464] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 1019.678464] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1019.678747] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 1020.635135] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1020.635392] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1020.635588] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1020.648964] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1020.649221] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1020.649383] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1020.649540] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1020.650974] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-215cb3ee-cb7c-4e7d-bb15-6e5677f98146 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1020.661880] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba27b96c-5c9b-4372-8a82-e470394b2a84 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1020.677384] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0469875d-9a74-4994-bc3c-d3758ce0dda6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1020.684088] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f79002e9-2265-4136-9823-55ef228fd6ee {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1020.714410] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180965MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1020.714578] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1020.714788] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1020.814466] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b0cc1acd-0e33-4547-a43b-549b3de81728 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1020.814685] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance eca7c94f-9bd1-4c15-988e-d4e33086ba40 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1020.814849] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ff4895f8-0374-4b34-a5e8-d3ba24f840a5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1020.815033] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ca18af6b-6b80-44f3-bd8b-f61e2596acd7 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1020.815211] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 02942b29-2343-441e-9e2b-aca2af57c849 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1020.818213] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ebd6c719-32ae-410c-a444-db7ad3f8a9c5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1020.818213] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1020.818213] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 411bafb7-ef72-4529-b3f3-0b4955f23788 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1020.818213] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ada70883-9bef-496e-a65f-ca84f6ee6211 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1020.818387] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1020.828026] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b271cd73-a148-48d6-bed9-6a99512457bd has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1020.841247] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fe967117-5b16-4b5e-aeb7-626e78fee756 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1020.857018] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 07fe1ba8-e5c2-421f-98db-680d90f39913 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1020.867520] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance a3a99c59-0c99-424e-b86c-5f58b494b4ef has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1020.882757] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 505f538c-051f-42e2-9da0-057e29bbb009 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1020.893403] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance a4bd5964-ee9e-44dc-a691-2382dff58c36 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1020.905507] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b800574b-cf7f-458b-ad5c-7c5c3dc5044f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1020.920551] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ee2343dc-52da-4327-afb2-83a774cb6fd0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1020.931485] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 017d212b-25f6-4b00-a1f4-090077a1fe72 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1020.942683] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 15a38322-5e08-4914-b609-a9b5941c9253 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1020.957105] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 0a502eb7-d333-4a4c-a75b-710bf7921871 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1020.971304] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9ad07396-2d8c-4277-b868-206e309631f0 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1020.982567] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1020.994658] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f6499ee5-cb24-44e1-9fe7-c19feeb732eb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1021.007026] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e7d86d5f-f92d-4891-b490-8d0fe37537fb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1021.018893] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c7081273-b937-4b9f-9369-79162ca200d1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1021.030490] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d3940c29-852a-427b-9027-aa4080150724 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1021.041342] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9065b97a-05a3-4b62-8f4a-418d5b0b209a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1021.052008] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e2fc8dbe-6458-4f5a-9aef-c44ea4d61b08 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1021.063426] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 6da6a17c-b133-4d3c-8b96-6317760a7b3b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1021.089195] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9f409285-631b-48f9-8b80-a3179c09a237 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1021.089195] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1021.089195] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1021.551422] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e36bbef-0b35-46a6-b0b9-beb45d88bd15 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1021.560249] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1cbf1ddd-8746-4b8f-bc5d-9442d53c2547 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1021.594832] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d0792c52-6f74-4028-bc42-0ad06f03de4a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1021.602587] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c7a16f4-4bf9-47b5-9ae1-75d50caefe13 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1021.616926] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1021.628178] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1021.644757] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1021.645247] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.930s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1022.645441] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1028.525060] env[67752]: WARNING oslo_vmware.rw_handles [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1028.525060] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1028.525060] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1028.525060] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1028.525060] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1028.525060] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1028.525060] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1028.525060] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1028.525060] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1028.525060] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1028.525060] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1028.525060] env[67752]: ERROR oslo_vmware.rw_handles [ 1028.525759] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/dac8e05e-4811-4029-a3a4-c5b400186b03/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1028.527690] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1028.527942] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Copying Virtual Disk [datastore2] vmware_temp/dac8e05e-4811-4029-a3a4-c5b400186b03/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/dac8e05e-4811-4029-a3a4-c5b400186b03/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1028.528251] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-c6ac3e99-f7fc-45ad-8343-d850b0ebb83d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1028.536189] env[67752]: DEBUG oslo_vmware.api [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Waiting for the task: (returnval){ [ 1028.536189] env[67752]: value = "task-3199706" [ 1028.536189] env[67752]: _type = "Task" [ 1028.536189] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1028.544444] env[67752]: DEBUG oslo_vmware.api [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Task: {'id': task-3199706, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1029.046378] env[67752]: DEBUG oslo_vmware.exceptions [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1029.046672] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1029.047243] env[67752]: ERROR nova.compute.manager [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1029.047243] env[67752]: Faults: ['InvalidArgument'] [ 1029.047243] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Traceback (most recent call last): [ 1029.047243] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1029.047243] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] yield resources [ 1029.047243] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1029.047243] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] self.driver.spawn(context, instance, image_meta, [ 1029.047243] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1029.047243] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1029.047243] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1029.047243] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] self._fetch_image_if_missing(context, vi) [ 1029.047243] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1029.047610] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] image_cache(vi, tmp_image_ds_loc) [ 1029.047610] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1029.047610] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] vm_util.copy_virtual_disk( [ 1029.047610] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1029.047610] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] session._wait_for_task(vmdk_copy_task) [ 1029.047610] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1029.047610] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] return self.wait_for_task(task_ref) [ 1029.047610] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1029.047610] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] return evt.wait() [ 1029.047610] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1029.047610] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] result = hub.switch() [ 1029.047610] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1029.047610] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] return self.greenlet.switch() [ 1029.047987] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1029.047987] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] self.f(*self.args, **self.kw) [ 1029.047987] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1029.047987] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] raise exceptions.translate_fault(task_info.error) [ 1029.047987] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1029.047987] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Faults: ['InvalidArgument'] [ 1029.047987] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] [ 1029.047987] env[67752]: INFO nova.compute.manager [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Terminating instance [ 1029.049187] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1029.049395] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1029.049755] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-be1f62b2-6fd8-4273-b51c-378846a372fc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.053388] env[67752]: DEBUG nova.compute.manager [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1029.053591] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1029.054324] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-097f2883-6adf-44c2-a834-74a913ef080a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.061333] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1029.062407] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-143c7ddc-0c27-4161-83a5-41667844b382 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.063950] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1029.064159] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1029.064868] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-589de3a7-d57c-464e-b37a-19c5c45df3c3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.070106] env[67752]: DEBUG oslo_vmware.api [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Waiting for the task: (returnval){ [ 1029.070106] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52bf547a-70b1-af0f-4c91-92192aae8354" [ 1029.070106] env[67752]: _type = "Task" [ 1029.070106] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1029.083211] env[67752]: DEBUG oslo_vmware.api [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52bf547a-70b1-af0f-4c91-92192aae8354, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1029.130409] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1029.130686] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1029.130916] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Deleting the datastore file [datastore2] b0cc1acd-0e33-4547-a43b-549b3de81728 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1029.131238] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-d17e15f8-8af9-4b3f-97bf-3e4252c3940f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.137972] env[67752]: DEBUG oslo_vmware.api [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Waiting for the task: (returnval){ [ 1029.137972] env[67752]: value = "task-3199709" [ 1029.137972] env[67752]: _type = "Task" [ 1029.137972] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1029.145854] env[67752]: DEBUG oslo_vmware.api [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Task: {'id': task-3199709, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1029.580384] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1029.580719] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Creating directory with path [datastore2] vmware_temp/cc9fab9d-a561-4c4b-b5ee-fc560f931cce/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1029.580936] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-3efd6207-4240-4f5b-bea2-8abcfd6066c7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.591629] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Created directory with path [datastore2] vmware_temp/cc9fab9d-a561-4c4b-b5ee-fc560f931cce/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1029.591819] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Fetch image to [datastore2] vmware_temp/cc9fab9d-a561-4c4b-b5ee-fc560f931cce/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1029.591987] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/cc9fab9d-a561-4c4b-b5ee-fc560f931cce/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1029.592712] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d471e4bc-5f49-44d6-8ddd-b2c132735310 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.599066] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c709bfe2-a6f8-47d7-b6d1-734ae705e822 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.607705] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e55981a-409f-46c1-942f-c37c7eaecded {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.637083] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4546173e-b39a-4c87-ade4-a5cb2b038353 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.647495] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-b4ae6c8d-3185-4a8c-b210-45afa2ae6704 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1029.649067] env[67752]: DEBUG oslo_vmware.api [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Task: {'id': task-3199709, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.076095} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1029.649302] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1029.649477] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1029.649675] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1029.649852] env[67752]: INFO nova.compute.manager [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1029.652014] env[67752]: DEBUG nova.compute.claims [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1029.652215] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1029.652432] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1029.740907] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1029.793778] env[67752]: DEBUG oslo_vmware.rw_handles [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/cc9fab9d-a561-4c4b-b5ee-fc560f931cce/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1029.853938] env[67752]: DEBUG oslo_vmware.rw_handles [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1029.854155] env[67752]: DEBUG oslo_vmware.rw_handles [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/cc9fab9d-a561-4c4b-b5ee-fc560f931cce/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1030.126439] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ce71ba4-016f-4b9a-9126-a54a073657da {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1030.134200] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b9886675-ab33-4853-a124-ab1c39b2e4e5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1030.163546] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e35d9728-cfcb-4f39-896b-83c41551f4f0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1030.171922] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-734925c7-f0ab-4c5b-8360-c1a2d6c9b980 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1030.184952] env[67752]: DEBUG nova.compute.provider_tree [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1030.212422] env[67752]: ERROR nova.scheduler.client.report [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [req-25b11919-ca47-4561-9790-9e0490dfaef6] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID c7141b98-bb6c-46bd-b37f-a8e913940199. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-25b11919-ca47-4561-9790-9e0490dfaef6"}]}: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1030.228172] env[67752]: DEBUG nova.scheduler.client.report [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Refreshing inventories for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:804}} [ 1030.242417] env[67752]: DEBUG nova.scheduler.client.report [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Updating ProviderTree inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:768}} [ 1030.242630] env[67752]: DEBUG nova.compute.provider_tree [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1030.253283] env[67752]: DEBUG nova.scheduler.client.report [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Refreshing aggregate associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, aggregates: None {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:813}} [ 1030.273078] env[67752]: DEBUG nova.scheduler.client.report [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Refreshing trait associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, traits: COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:825}} [ 1030.634023] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-114f6ec5-1c5a-4398-9282-e3f4a57270a6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1030.641521] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-77fa230b-a7d4-4637-8111-2cdec5261525 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1030.671195] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b719528-4958-4357-a855-be7df3011730 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1030.677715] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3f04a591-eff6-4887-9b3b-16c6cd863ef7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1030.690027] env[67752]: DEBUG nova.compute.provider_tree [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1030.724985] env[67752]: DEBUG nova.scheduler.client.report [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Updated inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with generation 71 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:957}} [ 1030.725261] env[67752]: DEBUG nova.compute.provider_tree [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Updating resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 generation from 71 to 72 during operation: update_inventory {{(pid=67752) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 1030.725441] env[67752]: DEBUG nova.compute.provider_tree [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1030.738496] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.086s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1030.739011] env[67752]: ERROR nova.compute.manager [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1030.739011] env[67752]: Faults: ['InvalidArgument'] [ 1030.739011] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Traceback (most recent call last): [ 1030.739011] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1030.739011] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] self.driver.spawn(context, instance, image_meta, [ 1030.739011] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1030.739011] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1030.739011] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1030.739011] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] self._fetch_image_if_missing(context, vi) [ 1030.739011] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1030.739011] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] image_cache(vi, tmp_image_ds_loc) [ 1030.739011] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1030.739353] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] vm_util.copy_virtual_disk( [ 1030.739353] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1030.739353] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] session._wait_for_task(vmdk_copy_task) [ 1030.739353] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1030.739353] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] return self.wait_for_task(task_ref) [ 1030.739353] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1030.739353] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] return evt.wait() [ 1030.739353] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1030.739353] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] result = hub.switch() [ 1030.739353] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1030.739353] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] return self.greenlet.switch() [ 1030.739353] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1030.739353] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] self.f(*self.args, **self.kw) [ 1030.739701] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1030.739701] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] raise exceptions.translate_fault(task_info.error) [ 1030.739701] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1030.739701] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Faults: ['InvalidArgument'] [ 1030.739701] env[67752]: ERROR nova.compute.manager [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] [ 1030.739816] env[67752]: DEBUG nova.compute.utils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1030.741163] env[67752]: DEBUG nova.compute.manager [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Build of instance b0cc1acd-0e33-4547-a43b-549b3de81728 was re-scheduled: A specified parameter was not correct: fileType [ 1030.741163] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1030.741532] env[67752]: DEBUG nova.compute.manager [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1030.741707] env[67752]: DEBUG nova.compute.manager [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1030.741865] env[67752]: DEBUG nova.compute.manager [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1030.742035] env[67752]: DEBUG nova.network.neutron [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1031.128620] env[67752]: DEBUG nova.network.neutron [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1031.138330] env[67752]: INFO nova.compute.manager [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Took 0.40 seconds to deallocate network for instance. [ 1031.265641] env[67752]: INFO nova.scheduler.client.report [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Deleted allocations for instance b0cc1acd-0e33-4547-a43b-549b3de81728 [ 1031.298570] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d9c94af4-fb62-4a3d-814c-84a54ed00039 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "b0cc1acd-0e33-4547-a43b-549b3de81728" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 425.053s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1031.301033] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9ee3a5c0-b3bd-4f15-9ac5-e4a45b38e7a6 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "b0cc1acd-0e33-4547-a43b-549b3de81728" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 25.170s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1031.301419] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9ee3a5c0-b3bd-4f15-9ac5-e4a45b38e7a6 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquiring lock "b0cc1acd-0e33-4547-a43b-549b3de81728-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1031.301782] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9ee3a5c0-b3bd-4f15-9ac5-e4a45b38e7a6 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "b0cc1acd-0e33-4547-a43b-549b3de81728-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1031.302084] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9ee3a5c0-b3bd-4f15-9ac5-e4a45b38e7a6 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "b0cc1acd-0e33-4547-a43b-549b3de81728-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1031.305691] env[67752]: INFO nova.compute.manager [None req-9ee3a5c0-b3bd-4f15-9ac5-e4a45b38e7a6 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Terminating instance [ 1031.308572] env[67752]: DEBUG nova.compute.manager [None req-9ee3a5c0-b3bd-4f15-9ac5-e4a45b38e7a6 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1031.308822] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-9ee3a5c0-b3bd-4f15-9ac5-e4a45b38e7a6 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1031.309255] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-7ec7bd5b-b2f2-4239-afa7-f475fbe60e44 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1031.314388] env[67752]: DEBUG nova.compute.manager [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1031.323430] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d91cb28b-1cc5-410d-b2a7-56146c404f60 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1031.355378] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-9ee3a5c0-b3bd-4f15-9ac5-e4a45b38e7a6 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b0cc1acd-0e33-4547-a43b-549b3de81728 could not be found. [ 1031.355510] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-9ee3a5c0-b3bd-4f15-9ac5-e4a45b38e7a6 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1031.355748] env[67752]: INFO nova.compute.manager [None req-9ee3a5c0-b3bd-4f15-9ac5-e4a45b38e7a6 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1031.355955] env[67752]: DEBUG oslo.service.loopingcall [None req-9ee3a5c0-b3bd-4f15-9ac5-e4a45b38e7a6 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1031.358447] env[67752]: DEBUG nova.compute.manager [-] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1031.358447] env[67752]: DEBUG nova.network.neutron [-] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1031.377424] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1031.377738] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1031.379337] env[67752]: INFO nova.compute.claims [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1031.396103] env[67752]: DEBUG nova.network.neutron [-] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1031.414498] env[67752]: INFO nova.compute.manager [-] [instance: b0cc1acd-0e33-4547-a43b-549b3de81728] Took 0.06 seconds to deallocate network for instance. [ 1031.538585] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9ee3a5c0-b3bd-4f15-9ac5-e4a45b38e7a6 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "b0cc1acd-0e33-4547-a43b-549b3de81728" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.238s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1031.843565] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94541693-6850-4176-85eb-2a1cf94f64cf {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1031.851142] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f1c30cc6-3d52-4039-a93c-300377984a7f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1031.880405] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c9323be9-3aeb-49fd-a9a5-4d32ab6401bf {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1031.887368] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec11ccd7-5b3e-4a98-9762-b3956abb758c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1031.900785] env[67752]: DEBUG nova.compute.provider_tree [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1031.909559] env[67752]: DEBUG nova.scheduler.client.report [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1031.926133] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.548s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1031.926627] env[67752]: DEBUG nova.compute.manager [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1032.674977] env[67752]: DEBUG nova.compute.utils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1032.676197] env[67752]: DEBUG nova.compute.manager [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1032.676374] env[67752]: DEBUG nova.network.neutron [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1032.687618] env[67752]: DEBUG nova.compute.manager [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1032.758576] env[67752]: DEBUG nova.compute.manager [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1032.778634] env[67752]: DEBUG nova.policy [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '103189dca7fa45909bbbf024b01128fd', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '0f84f46b9f074545b495c5a053cbd092', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1032.785203] env[67752]: DEBUG nova.virt.hardware [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1032.785498] env[67752]: DEBUG nova.virt.hardware [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1032.785740] env[67752]: DEBUG nova.virt.hardware [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1032.785975] env[67752]: DEBUG nova.virt.hardware [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1032.786178] env[67752]: DEBUG nova.virt.hardware [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1032.786375] env[67752]: DEBUG nova.virt.hardware [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1032.786624] env[67752]: DEBUG nova.virt.hardware [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1032.786816] env[67752]: DEBUG nova.virt.hardware [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1032.787046] env[67752]: DEBUG nova.virt.hardware [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1032.787257] env[67752]: DEBUG nova.virt.hardware [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1032.787563] env[67752]: DEBUG nova.virt.hardware [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1032.788817] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-be876682-b7ca-4f31-ba7a-4639deed33f9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1032.796372] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2d94abc1-ef71-4cc7-94a7-d4029c27904a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1033.128982] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7ce83be0-0ebf-45d2-b9d2-c95afb5fb37f tempest-ServerShowV254Test-2141823222 tempest-ServerShowV254Test-2141823222-project-member] Acquiring lock "3b3fa52f-9512-4de3-8029-5cbafed0b1ae" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1033.128982] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7ce83be0-0ebf-45d2-b9d2-c95afb5fb37f tempest-ServerShowV254Test-2141823222 tempest-ServerShowV254Test-2141823222-project-member] Lock "3b3fa52f-9512-4de3-8029-5cbafed0b1ae" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1033.273085] env[67752]: DEBUG nova.network.neutron [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Successfully created port: e9f31b14-e8ba-40f5-8a83-6e4a6779c8f9 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1034.477502] env[67752]: DEBUG nova.compute.manager [req-5c16c47b-97f6-4c34-adfe-f40e5b31ff80 req-7cf6f327-004b-4c12-afd8-05bf7da01e7f service nova] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Received event network-vif-plugged-e9f31b14-e8ba-40f5-8a83-6e4a6779c8f9 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1034.477794] env[67752]: DEBUG oslo_concurrency.lockutils [req-5c16c47b-97f6-4c34-adfe-f40e5b31ff80 req-7cf6f327-004b-4c12-afd8-05bf7da01e7f service nova] Acquiring lock "b271cd73-a148-48d6-bed9-6a99512457bd-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1034.477908] env[67752]: DEBUG oslo_concurrency.lockutils [req-5c16c47b-97f6-4c34-adfe-f40e5b31ff80 req-7cf6f327-004b-4c12-afd8-05bf7da01e7f service nova] Lock "b271cd73-a148-48d6-bed9-6a99512457bd-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1034.478238] env[67752]: DEBUG oslo_concurrency.lockutils [req-5c16c47b-97f6-4c34-adfe-f40e5b31ff80 req-7cf6f327-004b-4c12-afd8-05bf7da01e7f service nova] Lock "b271cd73-a148-48d6-bed9-6a99512457bd-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1034.478444] env[67752]: DEBUG nova.compute.manager [req-5c16c47b-97f6-4c34-adfe-f40e5b31ff80 req-7cf6f327-004b-4c12-afd8-05bf7da01e7f service nova] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] No waiting events found dispatching network-vif-plugged-e9f31b14-e8ba-40f5-8a83-6e4a6779c8f9 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1034.478619] env[67752]: WARNING nova.compute.manager [req-5c16c47b-97f6-4c34-adfe-f40e5b31ff80 req-7cf6f327-004b-4c12-afd8-05bf7da01e7f service nova] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Received unexpected event network-vif-plugged-e9f31b14-e8ba-40f5-8a83-6e4a6779c8f9 for instance with vm_state building and task_state spawning. [ 1034.592135] env[67752]: DEBUG nova.network.neutron [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Successfully updated port: e9f31b14-e8ba-40f5-8a83-6e4a6779c8f9 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1034.608339] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Acquiring lock "refresh_cache-b271cd73-a148-48d6-bed9-6a99512457bd" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1034.608502] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Acquired lock "refresh_cache-b271cd73-a148-48d6-bed9-6a99512457bd" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1034.608660] env[67752]: DEBUG nova.network.neutron [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1034.710463] env[67752]: DEBUG nova.network.neutron [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1034.926627] env[67752]: DEBUG nova.network.neutron [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Updating instance_info_cache with network_info: [{"id": "e9f31b14-e8ba-40f5-8a83-6e4a6779c8f9", "address": "fa:16:3e:17:e0:fb", "network": {"id": "f58709d0-25bb-4e87-824d-62ed007bb9a8", "bridge": "br-int", "label": "tempest-ServerMetadataNegativeTestJSON-545680887-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.13", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "0f84f46b9f074545b495c5a053cbd092", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "e9fa4744-8702-4973-b911-ee18192a3e4b", "external-id": "nsx-vlan-transportzone-318", "segmentation_id": 318, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape9f31b14-e8", "ovs_interfaceid": "e9f31b14-e8ba-40f5-8a83-6e4a6779c8f9", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1034.938351] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Releasing lock "refresh_cache-b271cd73-a148-48d6-bed9-6a99512457bd" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1034.939786] env[67752]: DEBUG nova.compute.manager [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Instance network_info: |[{"id": "e9f31b14-e8ba-40f5-8a83-6e4a6779c8f9", "address": "fa:16:3e:17:e0:fb", "network": {"id": "f58709d0-25bb-4e87-824d-62ed007bb9a8", "bridge": "br-int", "label": "tempest-ServerMetadataNegativeTestJSON-545680887-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.13", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "0f84f46b9f074545b495c5a053cbd092", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "e9fa4744-8702-4973-b911-ee18192a3e4b", "external-id": "nsx-vlan-transportzone-318", "segmentation_id": 318, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape9f31b14-e8", "ovs_interfaceid": "e9f31b14-e8ba-40f5-8a83-6e4a6779c8f9", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1034.939920] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:17:e0:fb', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'e9fa4744-8702-4973-b911-ee18192a3e4b', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'e9f31b14-e8ba-40f5-8a83-6e4a6779c8f9', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1034.946715] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Creating folder: Project (0f84f46b9f074545b495c5a053cbd092). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1034.947210] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-1fe95412-f245-4b1a-8453-720028dffcaf {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1034.960100] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Created folder: Project (0f84f46b9f074545b495c5a053cbd092) in parent group-v639722. [ 1034.960297] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Creating folder: Instances. Parent ref: group-v639788. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1034.960532] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-d6c39ef8-0395-49ab-972c-089a216e4edc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1034.969168] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Created folder: Instances in parent group-v639788. [ 1034.969392] env[67752]: DEBUG oslo.service.loopingcall [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1034.969594] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1034.969790] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-0417fcf3-f76d-4dd9-a574-8057a790dd83 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1034.987641] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1034.987641] env[67752]: value = "task-3199714" [ 1034.987641] env[67752]: _type = "Task" [ 1034.987641] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1034.995123] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199714, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1035.498212] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199714, 'name': CreateVM_Task, 'duration_secs': 0.285519} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1035.498506] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1035.498994] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1035.499173] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1035.499541] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1035.499746] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-a3ea0a05-f4b8-4baa-82d1-bf0e2aac05f0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1035.503899] env[67752]: DEBUG oslo_vmware.api [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Waiting for the task: (returnval){ [ 1035.503899] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52deb0ca-8db2-38ac-8116-eda10f7eeb20" [ 1035.503899] env[67752]: _type = "Task" [ 1035.503899] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1035.513987] env[67752]: DEBUG oslo_vmware.api [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52deb0ca-8db2-38ac-8116-eda10f7eeb20, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1036.014614] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1036.014851] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1036.015071] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1036.576027] env[67752]: DEBUG nova.compute.manager [req-d52eb6d9-72c9-4037-a753-12f6fafe3bfe req-120461c1-f4f5-4df9-bdc1-a2c920937a5a service nova] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Received event network-changed-e9f31b14-e8ba-40f5-8a83-6e4a6779c8f9 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1036.576310] env[67752]: DEBUG nova.compute.manager [req-d52eb6d9-72c9-4037-a753-12f6fafe3bfe req-120461c1-f4f5-4df9-bdc1-a2c920937a5a service nova] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Refreshing instance network info cache due to event network-changed-e9f31b14-e8ba-40f5-8a83-6e4a6779c8f9. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1036.576536] env[67752]: DEBUG oslo_concurrency.lockutils [req-d52eb6d9-72c9-4037-a753-12f6fafe3bfe req-120461c1-f4f5-4df9-bdc1-a2c920937a5a service nova] Acquiring lock "refresh_cache-b271cd73-a148-48d6-bed9-6a99512457bd" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1036.576684] env[67752]: DEBUG oslo_concurrency.lockutils [req-d52eb6d9-72c9-4037-a753-12f6fafe3bfe req-120461c1-f4f5-4df9-bdc1-a2c920937a5a service nova] Acquired lock "refresh_cache-b271cd73-a148-48d6-bed9-6a99512457bd" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1036.576845] env[67752]: DEBUG nova.network.neutron [req-d52eb6d9-72c9-4037-a753-12f6fafe3bfe req-120461c1-f4f5-4df9-bdc1-a2c920937a5a service nova] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Refreshing network info cache for port e9f31b14-e8ba-40f5-8a83-6e4a6779c8f9 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1037.005231] env[67752]: DEBUG nova.network.neutron [req-d52eb6d9-72c9-4037-a753-12f6fafe3bfe req-120461c1-f4f5-4df9-bdc1-a2c920937a5a service nova] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Updated VIF entry in instance network info cache for port e9f31b14-e8ba-40f5-8a83-6e4a6779c8f9. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1037.005612] env[67752]: DEBUG nova.network.neutron [req-d52eb6d9-72c9-4037-a753-12f6fafe3bfe req-120461c1-f4f5-4df9-bdc1-a2c920937a5a service nova] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Updating instance_info_cache with network_info: [{"id": "e9f31b14-e8ba-40f5-8a83-6e4a6779c8f9", "address": "fa:16:3e:17:e0:fb", "network": {"id": "f58709d0-25bb-4e87-824d-62ed007bb9a8", "bridge": "br-int", "label": "tempest-ServerMetadataNegativeTestJSON-545680887-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.13", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "0f84f46b9f074545b495c5a053cbd092", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "e9fa4744-8702-4973-b911-ee18192a3e4b", "external-id": "nsx-vlan-transportzone-318", "segmentation_id": 318, "bound_drivers": {"0": "nsxv3"}}, "devname": "tape9f31b14-e8", "ovs_interfaceid": "e9f31b14-e8ba-40f5-8a83-6e4a6779c8f9", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1037.014798] env[67752]: DEBUG oslo_concurrency.lockutils [req-d52eb6d9-72c9-4037-a753-12f6fafe3bfe req-120461c1-f4f5-4df9-bdc1-a2c920937a5a service nova] Releasing lock "refresh_cache-b271cd73-a148-48d6-bed9-6a99512457bd" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1038.868224] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8b4be1c0-6841-4ccd-a60c-9978f6ce6b69 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Acquiring lock "b271cd73-a148-48d6-bed9-6a99512457bd" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1041.583694] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Acquiring lock "7c8fc8d3-1f51-4793-8954-93451ebffa74" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1041.584092] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Lock "7c8fc8d3-1f51-4793-8954-93451ebffa74" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1047.306834] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "3d893e80-bbbb-4e58-bf1f-1edd4bccfd05" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1047.307301] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "3d893e80-bbbb-4e58-bf1f-1edd4bccfd05" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1051.571677] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Acquiring lock "a8bd310e-530a-46ab-add7-1b827ea5f399" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1051.572256] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Lock "a8bd310e-530a-46ab-add7-1b827ea5f399" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1054.254145] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9889c44c-b170-46af-9567-8a28eff30545 tempest-ServersAdminTestJSON-657333820 tempest-ServersAdminTestJSON-657333820-project-member] Acquiring lock "ded2bdfd-71c9-4af4-b595-98ac91fc8203" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1054.254562] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9889c44c-b170-46af-9567-8a28eff30545 tempest-ServersAdminTestJSON-657333820 tempest-ServersAdminTestJSON-657333820-project-member] Lock "ded2bdfd-71c9-4af4-b595-98ac91fc8203" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1054.951554] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bfcbcd14-66a8-45e4-a9a8-bdc99133a79a tempest-ServersAdminTestJSON-657333820 tempest-ServersAdminTestJSON-657333820-project-member] Acquiring lock "3d423978-c9cf-476f-9df8-d78206dff460" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1054.951666] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bfcbcd14-66a8-45e4-a9a8-bdc99133a79a tempest-ServersAdminTestJSON-657333820 tempest-ServersAdminTestJSON-657333820-project-member] Lock "3d423978-c9cf-476f-9df8-d78206dff460" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1056.928712] env[67752]: DEBUG oslo_concurrency.lockutils [None req-271904e9-41e9-43d6-b9d8-dd2f1c1e1a11 tempest-ServerShowV257Test-560095070 tempest-ServerShowV257Test-560095070-project-member] Acquiring lock "7c9546af-a9b8-4808-b15e-2ff9d2c12df7" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1056.928712] env[67752]: DEBUG oslo_concurrency.lockutils [None req-271904e9-41e9-43d6-b9d8-dd2f1c1e1a11 tempest-ServerShowV257Test-560095070 tempest-ServerShowV257Test-560095070-project-member] Lock "7c9546af-a9b8-4808-b15e-2ff9d2c12df7" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1058.059877] env[67752]: DEBUG oslo_concurrency.lockutils [None req-50f8be6d-a6fa-472c-a580-8749396bef71 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "78e8ccdd-e5ed-4b87-af85-498ca755561b" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1058.060429] env[67752]: DEBUG oslo_concurrency.lockutils [None req-50f8be6d-a6fa-472c-a580-8749396bef71 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "78e8ccdd-e5ed-4b87-af85-498ca755561b" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1059.511542] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a19e1d7-beb1-47b1-89d6-4b4427bf93cb tempest-SecurityGroupsTestJSON-1418396030 tempest-SecurityGroupsTestJSON-1418396030-project-member] Acquiring lock "ffa5bab7-fce4-4fa7-9e43-c4fc35ee6c45" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1059.511840] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a19e1d7-beb1-47b1-89d6-4b4427bf93cb tempest-SecurityGroupsTestJSON-1418396030 tempest-SecurityGroupsTestJSON-1418396030-project-member] Lock "ffa5bab7-fce4-4fa7-9e43-c4fc35ee6c45" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1061.002321] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3fd7bad-0c43-4856-bf43-3cbd4db3b6ee tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] Acquiring lock "2573ebf9-e18d-4b07-bde6-459a2466fe0d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1061.002637] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3fd7bad-0c43-4856-bf43-3cbd4db3b6ee tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] Lock "2573ebf9-e18d-4b07-bde6-459a2466fe0d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1077.630778] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1077.634484] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1078.180905] env[67752]: WARNING oslo_vmware.rw_handles [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1078.180905] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1078.180905] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1078.180905] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1078.180905] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1078.180905] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1078.180905] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1078.180905] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1078.180905] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1078.180905] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1078.180905] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1078.180905] env[67752]: ERROR oslo_vmware.rw_handles [ 1078.181421] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/cc9fab9d-a561-4c4b-b5ee-fc560f931cce/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1078.183657] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1078.183921] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Copying Virtual Disk [datastore2] vmware_temp/cc9fab9d-a561-4c4b-b5ee-fc560f931cce/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/cc9fab9d-a561-4c4b-b5ee-fc560f931cce/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1078.184218] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-29a2e760-41b7-43c8-81da-33472c73f3e4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1078.193044] env[67752]: DEBUG oslo_vmware.api [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Waiting for the task: (returnval){ [ 1078.193044] env[67752]: value = "task-3199717" [ 1078.193044] env[67752]: _type = "Task" [ 1078.193044] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1078.200775] env[67752]: DEBUG oslo_vmware.api [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Task: {'id': task-3199717, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1078.704758] env[67752]: DEBUG oslo_vmware.exceptions [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1078.705058] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1078.705611] env[67752]: ERROR nova.compute.manager [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1078.705611] env[67752]: Faults: ['InvalidArgument'] [ 1078.705611] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Traceback (most recent call last): [ 1078.705611] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1078.705611] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] yield resources [ 1078.705611] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1078.705611] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] self.driver.spawn(context, instance, image_meta, [ 1078.705611] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1078.705611] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1078.705611] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1078.705611] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] self._fetch_image_if_missing(context, vi) [ 1078.705611] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1078.705989] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] image_cache(vi, tmp_image_ds_loc) [ 1078.705989] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1078.705989] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] vm_util.copy_virtual_disk( [ 1078.705989] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1078.705989] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] session._wait_for_task(vmdk_copy_task) [ 1078.705989] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1078.705989] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] return self.wait_for_task(task_ref) [ 1078.705989] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1078.705989] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] return evt.wait() [ 1078.705989] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1078.705989] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] result = hub.switch() [ 1078.705989] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1078.705989] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] return self.greenlet.switch() [ 1078.706517] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1078.706517] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] self.f(*self.args, **self.kw) [ 1078.706517] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1078.706517] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] raise exceptions.translate_fault(task_info.error) [ 1078.706517] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1078.706517] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Faults: ['InvalidArgument'] [ 1078.706517] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] [ 1078.706517] env[67752]: INFO nova.compute.manager [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Terminating instance [ 1078.707702] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1078.707915] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1078.708949] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-9031cd53-efff-4531-ada2-77c0656c8009 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1078.711337] env[67752]: DEBUG nova.compute.manager [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1078.711531] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1078.712316] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dce1040e-591f-4e49-8ab1-fc39f477df56 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1078.719484] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1078.719724] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-0546dd26-fba8-42dd-9706-cb460701df3d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1078.722258] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1078.722376] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1078.723381] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c2aa2c68-b4ce-4f3d-ad3e-31bdbd815b7e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1078.728174] env[67752]: DEBUG oslo_vmware.api [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Waiting for the task: (returnval){ [ 1078.728174] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]529707a2-f824-9c3c-1c04-97395ecc4104" [ 1078.728174] env[67752]: _type = "Task" [ 1078.728174] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1078.735630] env[67752]: DEBUG oslo_vmware.api [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]529707a2-f824-9c3c-1c04-97395ecc4104, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1078.784436] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1078.784723] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1078.784955] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Deleting the datastore file [datastore2] eca7c94f-9bd1-4c15-988e-d4e33086ba40 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1078.785358] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-c06323a8-b031-4b9d-9603-3186d741f43a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1078.792202] env[67752]: DEBUG oslo_vmware.api [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Waiting for the task: (returnval){ [ 1078.792202] env[67752]: value = "task-3199719" [ 1078.792202] env[67752]: _type = "Task" [ 1078.792202] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1078.800866] env[67752]: DEBUG oslo_vmware.api [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Task: {'id': task-3199719, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1079.238276] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1079.238542] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Creating directory with path [datastore2] vmware_temp/d33fef25-1421-419d-b12d-69ca77505c06/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1079.238778] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-6b569f2c-0bb1-4bbb-a59c-07334c920b1b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1079.250260] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Created directory with path [datastore2] vmware_temp/d33fef25-1421-419d-b12d-69ca77505c06/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1079.250459] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Fetch image to [datastore2] vmware_temp/d33fef25-1421-419d-b12d-69ca77505c06/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1079.250627] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/d33fef25-1421-419d-b12d-69ca77505c06/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1079.251393] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c19b7d13-309c-416d-a18b-df2c3b618f2c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1079.258221] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f097612b-0e10-4d96-9418-c03fe4c3d67a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1079.267212] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9a1909d9-a053-416d-9295-360d2b903bfe {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1079.299723] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dae32062-621d-48a3-abd6-69c79ee56c58 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1079.307525] env[67752]: DEBUG oslo_vmware.api [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Task: {'id': task-3199719, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.07377} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1079.308948] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1079.309253] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1079.309493] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1079.309703] env[67752]: INFO nova.compute.manager [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1079.311566] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-fe386f5f-4dde-46b9-ae98-76d817a94641 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1079.313467] env[67752]: DEBUG nova.compute.claims [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1079.313698] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1079.313969] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1079.337305] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1079.392790] env[67752]: DEBUG oslo_vmware.rw_handles [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/d33fef25-1421-419d-b12d-69ca77505c06/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1079.456915] env[67752]: DEBUG oslo_vmware.rw_handles [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1079.457221] env[67752]: DEBUG oslo_vmware.rw_handles [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/d33fef25-1421-419d-b12d-69ca77505c06/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1079.634642] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1079.634847] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1079.634992] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 1079.758340] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a15cd41e-a4f3-4933-8d9f-7becb50044f8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1079.766400] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5e58ee04-ba8b-4dd7-b2d3-27172d187d28 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1079.799730] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa43ff94-ec80-478e-b739-04e0a38c505b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1079.808079] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bb8d9586-9c36-4d83-82ac-24faf5939e22 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1079.822473] env[67752]: DEBUG nova.compute.provider_tree [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1079.848027] env[67752]: ERROR nova.scheduler.client.report [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [req-258c2488-a873-46ae-b5b1-d0b55249d055] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID c7141b98-bb6c-46bd-b37f-a8e913940199. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-258c2488-a873-46ae-b5b1-d0b55249d055"}]}: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1079.866272] env[67752]: DEBUG nova.scheduler.client.report [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Refreshing inventories for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:804}} [ 1079.881110] env[67752]: DEBUG nova.scheduler.client.report [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Updating ProviderTree inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:768}} [ 1079.881336] env[67752]: DEBUG nova.compute.provider_tree [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1079.894761] env[67752]: DEBUG nova.scheduler.client.report [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Refreshing aggregate associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, aggregates: None {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:813}} [ 1079.915045] env[67752]: DEBUG nova.scheduler.client.report [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Refreshing trait associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, traits: COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:825}} [ 1080.209863] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b3db6d2-5650-4bcb-b213-2ff79bf96874 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1080.217492] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-39acea79-07cd-4eb5-81f7-f3505474aab6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1080.246716] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-84d4c0d0-2a55-42d7-82a0-d8bf98d7d86c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1080.253511] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b7acffaa-4afd-4209-85bb-58f3ce04fdf0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1080.266189] env[67752]: DEBUG nova.compute.provider_tree [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1080.300107] env[67752]: DEBUG nova.scheduler.client.report [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Updated inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with generation 82 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:957}} [ 1080.300381] env[67752]: DEBUG nova.compute.provider_tree [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Updating resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 generation from 82 to 83 during operation: update_inventory {{(pid=67752) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 1080.300562] env[67752]: DEBUG nova.compute.provider_tree [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1080.316792] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 1.003s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1080.317307] env[67752]: ERROR nova.compute.manager [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1080.317307] env[67752]: Faults: ['InvalidArgument'] [ 1080.317307] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Traceback (most recent call last): [ 1080.317307] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1080.317307] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] self.driver.spawn(context, instance, image_meta, [ 1080.317307] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1080.317307] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1080.317307] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1080.317307] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] self._fetch_image_if_missing(context, vi) [ 1080.317307] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1080.317307] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] image_cache(vi, tmp_image_ds_loc) [ 1080.317307] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1080.317669] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] vm_util.copy_virtual_disk( [ 1080.317669] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1080.317669] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] session._wait_for_task(vmdk_copy_task) [ 1080.317669] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1080.317669] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] return self.wait_for_task(task_ref) [ 1080.317669] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1080.317669] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] return evt.wait() [ 1080.317669] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1080.317669] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] result = hub.switch() [ 1080.317669] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1080.317669] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] return self.greenlet.switch() [ 1080.317669] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1080.317669] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] self.f(*self.args, **self.kw) [ 1080.318041] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1080.318041] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] raise exceptions.translate_fault(task_info.error) [ 1080.318041] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1080.318041] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Faults: ['InvalidArgument'] [ 1080.318041] env[67752]: ERROR nova.compute.manager [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] [ 1080.318041] env[67752]: DEBUG nova.compute.utils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1080.319599] env[67752]: DEBUG nova.compute.manager [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Build of instance eca7c94f-9bd1-4c15-988e-d4e33086ba40 was re-scheduled: A specified parameter was not correct: fileType [ 1080.319599] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1080.319969] env[67752]: DEBUG nova.compute.manager [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1080.320164] env[67752]: DEBUG nova.compute.manager [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1080.320336] env[67752]: DEBUG nova.compute.manager [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1080.320500] env[67752]: DEBUG nova.network.neutron [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1080.635560] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1080.635755] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 1080.635882] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 1080.664601] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1080.664812] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1080.664965] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1080.667278] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1080.667459] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1080.667668] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1080.667722] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1080.667847] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1080.668132] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1080.668132] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 1080.696156] env[67752]: DEBUG nova.network.neutron [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1080.713368] env[67752]: INFO nova.compute.manager [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Took 0.39 seconds to deallocate network for instance. [ 1080.827479] env[67752]: INFO nova.scheduler.client.report [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Deleted allocations for instance eca7c94f-9bd1-4c15-988e-d4e33086ba40 [ 1080.851282] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4edd2637-039d-483e-88b8-f9993a3dd4da tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Lock "eca7c94f-9bd1-4c15-988e-d4e33086ba40" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 473.440s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1080.851715] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e8016b21-7c3d-4f6a-9a56-cf803514cb24 tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Lock "eca7c94f-9bd1-4c15-988e-d4e33086ba40" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 272.565s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1080.853176] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e8016b21-7c3d-4f6a-9a56-cf803514cb24 tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Acquiring lock "eca7c94f-9bd1-4c15-988e-d4e33086ba40-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1080.853176] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e8016b21-7c3d-4f6a-9a56-cf803514cb24 tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Lock "eca7c94f-9bd1-4c15-988e-d4e33086ba40-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1080.853176] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e8016b21-7c3d-4f6a-9a56-cf803514cb24 tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Lock "eca7c94f-9bd1-4c15-988e-d4e33086ba40-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1080.854838] env[67752]: INFO nova.compute.manager [None req-e8016b21-7c3d-4f6a-9a56-cf803514cb24 tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Terminating instance [ 1080.856681] env[67752]: DEBUG nova.compute.manager [None req-e8016b21-7c3d-4f6a-9a56-cf803514cb24 tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1080.856876] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-e8016b21-7c3d-4f6a-9a56-cf803514cb24 tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1080.857368] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5a2b88d9-ecd0-4f79-881e-757e2aefa43e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1080.869948] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e8ee2adc-32a8-4e83-8b1d-28aa5a2db560 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1080.877369] env[67752]: DEBUG nova.compute.manager [None req-b416a339-6230-4165-9968-d38a5aabc674 tempest-AttachVolumeShelveTestJSON-812146230 tempest-AttachVolumeShelveTestJSON-812146230-project-member] [instance: fe967117-5b16-4b5e-aeb7-626e78fee756] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1080.900074] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-e8016b21-7c3d-4f6a-9a56-cf803514cb24 tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance eca7c94f-9bd1-4c15-988e-d4e33086ba40 could not be found. [ 1080.900074] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-e8016b21-7c3d-4f6a-9a56-cf803514cb24 tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1080.900074] env[67752]: INFO nova.compute.manager [None req-e8016b21-7c3d-4f6a-9a56-cf803514cb24 tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1080.900074] env[67752]: DEBUG oslo.service.loopingcall [None req-e8016b21-7c3d-4f6a-9a56-cf803514cb24 tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1080.900074] env[67752]: DEBUG nova.compute.manager [-] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1080.900397] env[67752]: DEBUG nova.network.neutron [-] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1080.908984] env[67752]: DEBUG nova.compute.manager [None req-b416a339-6230-4165-9968-d38a5aabc674 tempest-AttachVolumeShelveTestJSON-812146230 tempest-AttachVolumeShelveTestJSON-812146230-project-member] [instance: fe967117-5b16-4b5e-aeb7-626e78fee756] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1080.937917] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b416a339-6230-4165-9968-d38a5aabc674 tempest-AttachVolumeShelveTestJSON-812146230 tempest-AttachVolumeShelveTestJSON-812146230-project-member] Lock "fe967117-5b16-4b5e-aeb7-626e78fee756" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 237.299s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1080.942954] env[67752]: DEBUG nova.network.neutron [-] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1080.948988] env[67752]: DEBUG nova.compute.manager [None req-6141b860-2aa0-406b-b112-0dc765b9153a tempest-ServersNegativeTestJSON-289807960 tempest-ServersNegativeTestJSON-289807960-project-member] [instance: 07fe1ba8-e5c2-421f-98db-680d90f39913] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1080.952484] env[67752]: INFO nova.compute.manager [-] [instance: eca7c94f-9bd1-4c15-988e-d4e33086ba40] Took 0.05 seconds to deallocate network for instance. [ 1080.974203] env[67752]: DEBUG nova.compute.manager [None req-6141b860-2aa0-406b-b112-0dc765b9153a tempest-ServersNegativeTestJSON-289807960 tempest-ServersNegativeTestJSON-289807960-project-member] [instance: 07fe1ba8-e5c2-421f-98db-680d90f39913] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1080.996401] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6141b860-2aa0-406b-b112-0dc765b9153a tempest-ServersNegativeTestJSON-289807960 tempest-ServersNegativeTestJSON-289807960-project-member] Lock "07fe1ba8-e5c2-421f-98db-680d90f39913" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 234.708s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1081.007962] env[67752]: DEBUG nova.compute.manager [None req-cd94167a-d25b-4a92-b001-dd91b9582577 tempest-ServerDiagnosticsNegativeTest-355792777 tempest-ServerDiagnosticsNegativeTest-355792777-project-member] [instance: a3a99c59-0c99-424e-b86c-5f58b494b4ef] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1081.031431] env[67752]: DEBUG nova.compute.manager [None req-cd94167a-d25b-4a92-b001-dd91b9582577 tempest-ServerDiagnosticsNegativeTest-355792777 tempest-ServerDiagnosticsNegativeTest-355792777-project-member] [instance: a3a99c59-0c99-424e-b86c-5f58b494b4ef] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1081.056163] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e8016b21-7c3d-4f6a-9a56-cf803514cb24 tempest-ServersTestManualDisk-1892838690 tempest-ServersTestManualDisk-1892838690-project-member] Lock "eca7c94f-9bd1-4c15-988e-d4e33086ba40" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.204s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1081.064620] env[67752]: DEBUG oslo_concurrency.lockutils [None req-cd94167a-d25b-4a92-b001-dd91b9582577 tempest-ServerDiagnosticsNegativeTest-355792777 tempest-ServerDiagnosticsNegativeTest-355792777-project-member] Lock "a3a99c59-0c99-424e-b86c-5f58b494b4ef" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 233.856s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1081.073087] env[67752]: DEBUG nova.compute.manager [None req-bbfd8929-3e44-4655-9216-fd5ca59163b4 tempest-ServerGroupTestJSON-598670425 tempest-ServerGroupTestJSON-598670425-project-member] [instance: 505f538c-051f-42e2-9da0-057e29bbb009] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1081.096388] env[67752]: DEBUG nova.compute.manager [None req-bbfd8929-3e44-4655-9216-fd5ca59163b4 tempest-ServerGroupTestJSON-598670425 tempest-ServerGroupTestJSON-598670425-project-member] [instance: 505f538c-051f-42e2-9da0-057e29bbb009] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1081.117085] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbfd8929-3e44-4655-9216-fd5ca59163b4 tempest-ServerGroupTestJSON-598670425 tempest-ServerGroupTestJSON-598670425-project-member] Lock "505f538c-051f-42e2-9da0-057e29bbb009" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 230.488s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1081.126140] env[67752]: DEBUG nova.compute.manager [None req-39ef8043-c1d5-4e53-8ea1-e3c88ba7ee9f tempest-ServerShowV247Test-1751066233 tempest-ServerShowV247Test-1751066233-project-member] [instance: a4bd5964-ee9e-44dc-a691-2382dff58c36] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1081.152313] env[67752]: DEBUG nova.compute.manager [None req-39ef8043-c1d5-4e53-8ea1-e3c88ba7ee9f tempest-ServerShowV247Test-1751066233 tempest-ServerShowV247Test-1751066233-project-member] [instance: a4bd5964-ee9e-44dc-a691-2382dff58c36] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1081.175820] env[67752]: DEBUG oslo_concurrency.lockutils [None req-39ef8043-c1d5-4e53-8ea1-e3c88ba7ee9f tempest-ServerShowV247Test-1751066233 tempest-ServerShowV247Test-1751066233-project-member] Lock "a4bd5964-ee9e-44dc-a691-2382dff58c36" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 229.967s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1081.186124] env[67752]: DEBUG nova.compute.manager [None req-e5c22520-22ec-47ed-9010-411eb42dd3f7 tempest-ServerShowV247Test-1751066233 tempest-ServerShowV247Test-1751066233-project-member] [instance: b800574b-cf7f-458b-ad5c-7c5c3dc5044f] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1081.213031] env[67752]: DEBUG nova.compute.manager [None req-e5c22520-22ec-47ed-9010-411eb42dd3f7 tempest-ServerShowV247Test-1751066233 tempest-ServerShowV247Test-1751066233-project-member] [instance: b800574b-cf7f-458b-ad5c-7c5c3dc5044f] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1081.271473] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e5c22520-22ec-47ed-9010-411eb42dd3f7 tempest-ServerShowV247Test-1751066233 tempest-ServerShowV247Test-1751066233-project-member] Lock "b800574b-cf7f-458b-ad5c-7c5c3dc5044f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 229.073s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1081.283223] env[67752]: DEBUG nova.compute.manager [None req-8631916d-5152-413b-96d2-ef7bb793d025 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: ee2343dc-52da-4327-afb2-83a774cb6fd0] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1081.315703] env[67752]: DEBUG nova.compute.manager [None req-8631916d-5152-413b-96d2-ef7bb793d025 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: ee2343dc-52da-4327-afb2-83a774cb6fd0] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1081.337643] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8631916d-5152-413b-96d2-ef7bb793d025 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Lock "ee2343dc-52da-4327-afb2-83a774cb6fd0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 228.143s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1081.346407] env[67752]: DEBUG nova.compute.manager [None req-cd49a92d-3ef3-44dd-82e4-e594546ff1c4 tempest-SecurityGroupsTestJSON-1418396030 tempest-SecurityGroupsTestJSON-1418396030-project-member] [instance: 017d212b-25f6-4b00-a1f4-090077a1fe72] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1081.371894] env[67752]: DEBUG nova.compute.manager [None req-cd49a92d-3ef3-44dd-82e4-e594546ff1c4 tempest-SecurityGroupsTestJSON-1418396030 tempest-SecurityGroupsTestJSON-1418396030-project-member] [instance: 017d212b-25f6-4b00-a1f4-090077a1fe72] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1081.392401] env[67752]: DEBUG oslo_concurrency.lockutils [None req-cd49a92d-3ef3-44dd-82e4-e594546ff1c4 tempest-SecurityGroupsTestJSON-1418396030 tempest-SecurityGroupsTestJSON-1418396030-project-member] Lock "017d212b-25f6-4b00-a1f4-090077a1fe72" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 224.228s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1081.401035] env[67752]: DEBUG nova.compute.manager [None req-d3bf8ccf-d395-49d2-9675-da151d59e62c tempest-ServersAaction247Test-556426769 tempest-ServersAaction247Test-556426769-project-member] [instance: 15a38322-5e08-4914-b609-a9b5941c9253] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1081.422800] env[67752]: DEBUG nova.compute.manager [None req-d3bf8ccf-d395-49d2-9675-da151d59e62c tempest-ServersAaction247Test-556426769 tempest-ServersAaction247Test-556426769-project-member] [instance: 15a38322-5e08-4914-b609-a9b5941c9253] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1081.442920] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d3bf8ccf-d395-49d2-9675-da151d59e62c tempest-ServersAaction247Test-556426769 tempest-ServersAaction247Test-556426769-project-member] Lock "15a38322-5e08-4914-b609-a9b5941c9253" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 223.316s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1081.450772] env[67752]: DEBUG nova.compute.manager [None req-c753e970-12bd-4719-b442-113eb87194b3 tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] [instance: 0a502eb7-d333-4a4c-a75b-710bf7921871] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1081.471790] env[67752]: DEBUG nova.compute.manager [None req-c753e970-12bd-4719-b442-113eb87194b3 tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] [instance: 0a502eb7-d333-4a4c-a75b-710bf7921871] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1081.490158] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c753e970-12bd-4719-b442-113eb87194b3 tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] Lock "0a502eb7-d333-4a4c-a75b-710bf7921871" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 218.235s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1081.497640] env[67752]: DEBUG nova.compute.manager [None req-7c5206e6-bc4f-4023-9741-e1f496e47c50 tempest-ServerActionsTestOtherA-1579097132 tempest-ServerActionsTestOtherA-1579097132-project-member] [instance: 9ad07396-2d8c-4277-b868-206e309631f0] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1081.518935] env[67752]: DEBUG nova.compute.manager [None req-7c5206e6-bc4f-4023-9741-e1f496e47c50 tempest-ServerActionsTestOtherA-1579097132 tempest-ServerActionsTestOtherA-1579097132-project-member] [instance: 9ad07396-2d8c-4277-b868-206e309631f0] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1081.539078] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c5206e6-bc4f-4023-9741-e1f496e47c50 tempest-ServerActionsTestOtherA-1579097132 tempest-ServerActionsTestOtherA-1579097132-project-member] Lock "9ad07396-2d8c-4277-b868-206e309631f0" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 208.380s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1081.548909] env[67752]: DEBUG nova.compute.manager [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1081.599907] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1081.600191] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1081.601705] env[67752]: INFO nova.compute.claims [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1081.634820] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1081.935773] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-389b1da4-c597-4222-bed4-d84dc2bca782 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1081.943517] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1dfb7181-9b05-4f6f-a8de-99e5d869e701 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1081.973822] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0409b58d-44c3-4cfd-9418-b2ba744b337e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1081.981382] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2078bf8d-8c1b-4a7a-afba-8dc415e9f4b3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1081.994500] env[67752]: DEBUG nova.compute.provider_tree [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1082.004576] env[67752]: DEBUG nova.scheduler.client.report [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1082.018292] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.418s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1082.018840] env[67752]: DEBUG nova.compute.manager [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1082.051898] env[67752]: DEBUG nova.compute.utils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1082.053429] env[67752]: DEBUG nova.compute.manager [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1082.053607] env[67752]: DEBUG nova.network.neutron [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1082.063719] env[67752]: DEBUG nova.compute.manager [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1082.125177] env[67752]: DEBUG nova.compute.manager [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1082.152219] env[67752]: DEBUG nova.virt.hardware [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1082.152476] env[67752]: DEBUG nova.virt.hardware [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1082.152635] env[67752]: DEBUG nova.virt.hardware [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1082.152815] env[67752]: DEBUG nova.virt.hardware [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1082.153095] env[67752]: DEBUG nova.virt.hardware [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1082.153265] env[67752]: DEBUG nova.virt.hardware [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1082.153477] env[67752]: DEBUG nova.virt.hardware [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1082.153641] env[67752]: DEBUG nova.virt.hardware [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1082.153804] env[67752]: DEBUG nova.virt.hardware [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1082.153982] env[67752]: DEBUG nova.virt.hardware [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1082.154147] env[67752]: DEBUG nova.virt.hardware [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1082.155053] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8349a391-81f8-4bec-ac7d-cf1029fbfcab {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1082.162961] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-36b1923a-e9ad-4001-934c-d841c930519e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1082.178395] env[67752]: DEBUG nova.policy [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '8e748e58d3be4d3ea3448b18df2d4636', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd2a7e0d1165045ad8464da81b5b63c96', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1082.635238] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1082.636281] env[67752]: DEBUG nova.network.neutron [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Successfully created port: fb750bdb-df9c-49ec-8506-b79b30f48424 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1082.638444] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1082.653036] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1082.653363] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1082.653634] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1082.653884] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1082.655587] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d01c0f37-f368-49ba-a95e-dbd41abb201d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1082.666029] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9ba6430-f9ec-49e6-99ff-fd1ff21a24ec {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1082.679577] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4917ac65-e680-49de-a5c4-a1d763c73616 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1082.685992] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6de9c108-f023-454a-905a-bf7859455d97 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1082.714656] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180992MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1082.714814] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1082.715030] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1082.795709] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ff4895f8-0374-4b34-a5e8-d3ba24f840a5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1082.795880] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ca18af6b-6b80-44f3-bd8b-f61e2596acd7 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1082.796024] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 02942b29-2343-441e-9e2b-aca2af57c849 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1082.796165] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ebd6c719-32ae-410c-a444-db7ad3f8a9c5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1082.796289] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1082.796436] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 411bafb7-ef72-4529-b3f3-0b4955f23788 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1082.796523] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ada70883-9bef-496e-a65f-ca84f6ee6211 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1082.796638] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1082.796753] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b271cd73-a148-48d6-bed9-6a99512457bd actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1082.796863] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1082.811986] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f6499ee5-cb24-44e1-9fe7-c19feeb732eb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1082.823021] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e7d86d5f-f92d-4891-b490-8d0fe37537fb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1082.833732] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c7081273-b937-4b9f-9369-79162ca200d1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1082.843258] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d3940c29-852a-427b-9027-aa4080150724 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1082.852697] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9065b97a-05a3-4b62-8f4a-418d5b0b209a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1082.863306] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e2fc8dbe-6458-4f5a-9aef-c44ea4d61b08 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1082.872690] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 6da6a17c-b133-4d3c-8b96-6317760a7b3b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1082.882214] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9f409285-631b-48f9-8b80-a3179c09a237 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1082.893523] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3b3fa52f-9512-4de3-8029-5cbafed0b1ae has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1082.906155] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 7c8fc8d3-1f51-4793-8954-93451ebffa74 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1082.918082] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1082.927939] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance a8bd310e-530a-46ab-add7-1b827ea5f399 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1082.938237] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ded2bdfd-71c9-4af4-b595-98ac91fc8203 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1082.949777] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3d423978-c9cf-476f-9df8-d78206dff460 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1082.961531] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 7c9546af-a9b8-4808-b15e-2ff9d2c12df7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1082.977940] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 78e8ccdd-e5ed-4b87-af85-498ca755561b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1082.985341] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ffa5bab7-fce4-4fa7-9e43-c4fc35ee6c45 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1082.996456] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 2573ebf9-e18d-4b07-bde6-459a2466fe0d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1082.996718] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1082.996867] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1083.515353] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5228fd28-9998-468a-921d-2d67e843ced5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1083.523446] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-099978b8-e8c8-4c4c-8aaa-886b2592c16b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1083.553175] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ae485f6a-3f10-4a31-a057-ed641e20c288 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1083.560356] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d3dfcdb2-1544-4365-ba46-1e5cf46ccfd2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1083.574269] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1083.584672] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1083.599914] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1083.600123] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.885s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1083.864535] env[67752]: DEBUG nova.compute.manager [req-d80f534a-ebbf-45ff-8e23-82cb88ae2240 req-38b50b20-83b8-45e8-a3ff-b6457a4a7ead service nova] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Received event network-vif-plugged-fb750bdb-df9c-49ec-8506-b79b30f48424 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1083.864535] env[67752]: DEBUG oslo_concurrency.lockutils [req-d80f534a-ebbf-45ff-8e23-82cb88ae2240 req-38b50b20-83b8-45e8-a3ff-b6457a4a7ead service nova] Acquiring lock "3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1083.864535] env[67752]: DEBUG oslo_concurrency.lockutils [req-d80f534a-ebbf-45ff-8e23-82cb88ae2240 req-38b50b20-83b8-45e8-a3ff-b6457a4a7ead service nova] Lock "3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1083.864535] env[67752]: DEBUG oslo_concurrency.lockutils [req-d80f534a-ebbf-45ff-8e23-82cb88ae2240 req-38b50b20-83b8-45e8-a3ff-b6457a4a7ead service nova] Lock "3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1083.864805] env[67752]: DEBUG nova.compute.manager [req-d80f534a-ebbf-45ff-8e23-82cb88ae2240 req-38b50b20-83b8-45e8-a3ff-b6457a4a7ead service nova] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] No waiting events found dispatching network-vif-plugged-fb750bdb-df9c-49ec-8506-b79b30f48424 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1083.864805] env[67752]: WARNING nova.compute.manager [req-d80f534a-ebbf-45ff-8e23-82cb88ae2240 req-38b50b20-83b8-45e8-a3ff-b6457a4a7ead service nova] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Received unexpected event network-vif-plugged-fb750bdb-df9c-49ec-8506-b79b30f48424 for instance with vm_state building and task_state spawning. [ 1083.868114] env[67752]: DEBUG nova.network.neutron [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Successfully updated port: fb750bdb-df9c-49ec-8506-b79b30f48424 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1083.877532] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Acquiring lock "refresh_cache-3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1083.877699] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Acquired lock "refresh_cache-3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1083.877824] env[67752]: DEBUG nova.network.neutron [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1083.962467] env[67752]: DEBUG nova.network.neutron [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1084.293681] env[67752]: DEBUG nova.network.neutron [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Updating instance_info_cache with network_info: [{"id": "fb750bdb-df9c-49ec-8506-b79b30f48424", "address": "fa:16:3e:87:47:61", "network": {"id": "4d6304e1-7719-4952-a599-ebac448d5b5f", "bridge": "br-int", "label": "tempest-InstanceActionsV221TestJSON-170293278-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.11", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d2a7e0d1165045ad8464da81b5b63c96", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "2d88bb07-f93c-45ca-bce7-230cb1f33833", "external-id": "nsx-vlan-transportzone-387", "segmentation_id": 387, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapfb750bdb-df", "ovs_interfaceid": "fb750bdb-df9c-49ec-8506-b79b30f48424", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1084.308209] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Releasing lock "refresh_cache-3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1084.308515] env[67752]: DEBUG nova.compute.manager [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Instance network_info: |[{"id": "fb750bdb-df9c-49ec-8506-b79b30f48424", "address": "fa:16:3e:87:47:61", "network": {"id": "4d6304e1-7719-4952-a599-ebac448d5b5f", "bridge": "br-int", "label": "tempest-InstanceActionsV221TestJSON-170293278-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.11", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d2a7e0d1165045ad8464da81b5b63c96", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "2d88bb07-f93c-45ca-bce7-230cb1f33833", "external-id": "nsx-vlan-transportzone-387", "segmentation_id": 387, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapfb750bdb-df", "ovs_interfaceid": "fb750bdb-df9c-49ec-8506-b79b30f48424", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1084.308923] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:87:47:61', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '2d88bb07-f93c-45ca-bce7-230cb1f33833', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'fb750bdb-df9c-49ec-8506-b79b30f48424', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1084.316342] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Creating folder: Project (d2a7e0d1165045ad8464da81b5b63c96). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1084.316809] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-0a9fcd2b-7218-437c-b079-bfd94d16925c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1084.328105] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Created folder: Project (d2a7e0d1165045ad8464da81b5b63c96) in parent group-v639722. [ 1084.328302] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Creating folder: Instances. Parent ref: group-v639791. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1084.328526] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-3696f01f-96e1-4633-a29c-2a492e43ec76 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1084.336924] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Created folder: Instances in parent group-v639791. [ 1084.337163] env[67752]: DEBUG oslo.service.loopingcall [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1084.337348] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1084.337540] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-a622ac0c-b84b-41f9-9196-c83c92c4454d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1084.355909] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1084.355909] env[67752]: value = "task-3199722" [ 1084.355909] env[67752]: _type = "Task" [ 1084.355909] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1084.363353] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199722, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1084.596896] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1084.866494] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199722, 'name': CreateVM_Task, 'duration_secs': 0.333182} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1084.866671] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1084.867415] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1084.867577] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1084.867888] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1084.868156] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-fd96f075-f35f-4fdc-9314-54a23f469177 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1084.872436] env[67752]: DEBUG oslo_vmware.api [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Waiting for the task: (returnval){ [ 1084.872436] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5226bf6c-dc80-a39e-e94a-2059a15f5e0f" [ 1084.872436] env[67752]: _type = "Task" [ 1084.872436] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1084.879637] env[67752]: DEBUG oslo_vmware.api [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5226bf6c-dc80-a39e-e94a-2059a15f5e0f, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1085.382774] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1085.383091] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1085.383214] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1085.935225] env[67752]: DEBUG nova.compute.manager [req-8ea9b435-f346-4d40-ad53-74374f2d760d req-69cd6aa4-e2a1-4b33-97ce-d9cb91345d7f service nova] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Received event network-changed-fb750bdb-df9c-49ec-8506-b79b30f48424 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1085.935225] env[67752]: DEBUG nova.compute.manager [req-8ea9b435-f346-4d40-ad53-74374f2d760d req-69cd6aa4-e2a1-4b33-97ce-d9cb91345d7f service nova] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Refreshing instance network info cache due to event network-changed-fb750bdb-df9c-49ec-8506-b79b30f48424. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1085.935225] env[67752]: DEBUG oslo_concurrency.lockutils [req-8ea9b435-f346-4d40-ad53-74374f2d760d req-69cd6aa4-e2a1-4b33-97ce-d9cb91345d7f service nova] Acquiring lock "refresh_cache-3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1085.935225] env[67752]: DEBUG oslo_concurrency.lockutils [req-8ea9b435-f346-4d40-ad53-74374f2d760d req-69cd6aa4-e2a1-4b33-97ce-d9cb91345d7f service nova] Acquired lock "refresh_cache-3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1085.935225] env[67752]: DEBUG nova.network.neutron [req-8ea9b435-f346-4d40-ad53-74374f2d760d req-69cd6aa4-e2a1-4b33-97ce-d9cb91345d7f service nova] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Refreshing network info cache for port fb750bdb-df9c-49ec-8506-b79b30f48424 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1086.512907] env[67752]: DEBUG nova.network.neutron [req-8ea9b435-f346-4d40-ad53-74374f2d760d req-69cd6aa4-e2a1-4b33-97ce-d9cb91345d7f service nova] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Updated VIF entry in instance network info cache for port fb750bdb-df9c-49ec-8506-b79b30f48424. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1086.513293] env[67752]: DEBUG nova.network.neutron [req-8ea9b435-f346-4d40-ad53-74374f2d760d req-69cd6aa4-e2a1-4b33-97ce-d9cb91345d7f service nova] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Updating instance_info_cache with network_info: [{"id": "fb750bdb-df9c-49ec-8506-b79b30f48424", "address": "fa:16:3e:87:47:61", "network": {"id": "4d6304e1-7719-4952-a599-ebac448d5b5f", "bridge": "br-int", "label": "tempest-InstanceActionsV221TestJSON-170293278-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.11", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d2a7e0d1165045ad8464da81b5b63c96", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "2d88bb07-f93c-45ca-bce7-230cb1f33833", "external-id": "nsx-vlan-transportzone-387", "segmentation_id": 387, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapfb750bdb-df", "ovs_interfaceid": "fb750bdb-df9c-49ec-8506-b79b30f48424", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1086.524215] env[67752]: DEBUG oslo_concurrency.lockutils [req-8ea9b435-f346-4d40-ad53-74374f2d760d req-69cd6aa4-e2a1-4b33-97ce-d9cb91345d7f service nova] Releasing lock "refresh_cache-3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1090.918386] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4094cbfb-3682-4575-957c-d8b65674b023 tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Acquiring lock "3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1091.597216] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Acquiring lock "9432cd04-c2df-428b-b31a-93d3ec0c9d1f" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1091.597405] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Lock "9432cd04-c2df-428b-b31a-93d3ec0c9d1f" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1128.198378] env[67752]: WARNING oslo_vmware.rw_handles [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1128.198378] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1128.198378] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1128.198378] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1128.198378] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1128.198378] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1128.198378] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1128.198378] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1128.198378] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1128.198378] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1128.198378] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1128.198378] env[67752]: ERROR oslo_vmware.rw_handles [ 1128.199094] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/d33fef25-1421-419d-b12d-69ca77505c06/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1128.200707] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1128.200944] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Copying Virtual Disk [datastore2] vmware_temp/d33fef25-1421-419d-b12d-69ca77505c06/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/d33fef25-1421-419d-b12d-69ca77505c06/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1128.201244] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-902b6ff9-c24a-4ddc-9461-a96e90ae6ea5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1128.209834] env[67752]: DEBUG oslo_vmware.api [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Waiting for the task: (returnval){ [ 1128.209834] env[67752]: value = "task-3199723" [ 1128.209834] env[67752]: _type = "Task" [ 1128.209834] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1128.218673] env[67752]: DEBUG oslo_vmware.api [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Task: {'id': task-3199723, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1128.720590] env[67752]: DEBUG oslo_vmware.exceptions [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1128.720891] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1128.721565] env[67752]: ERROR nova.compute.manager [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1128.721565] env[67752]: Faults: ['InvalidArgument'] [ 1128.721565] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Traceback (most recent call last): [ 1128.721565] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1128.721565] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] yield resources [ 1128.721565] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1128.721565] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] self.driver.spawn(context, instance, image_meta, [ 1128.721565] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1128.721565] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1128.721565] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1128.721565] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] self._fetch_image_if_missing(context, vi) [ 1128.721565] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1128.721914] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] image_cache(vi, tmp_image_ds_loc) [ 1128.721914] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1128.721914] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] vm_util.copy_virtual_disk( [ 1128.721914] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1128.721914] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] session._wait_for_task(vmdk_copy_task) [ 1128.721914] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1128.721914] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] return self.wait_for_task(task_ref) [ 1128.721914] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1128.721914] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] return evt.wait() [ 1128.721914] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1128.721914] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] result = hub.switch() [ 1128.721914] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1128.721914] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] return self.greenlet.switch() [ 1128.722344] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1128.722344] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] self.f(*self.args, **self.kw) [ 1128.722344] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1128.722344] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] raise exceptions.translate_fault(task_info.error) [ 1128.722344] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1128.722344] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Faults: ['InvalidArgument'] [ 1128.722344] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] [ 1128.722344] env[67752]: INFO nova.compute.manager [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Terminating instance [ 1128.723581] env[67752]: DEBUG oslo_concurrency.lockutils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1128.723842] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1128.724095] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-904eb352-9998-4242-8705-a8af0049981e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1128.726194] env[67752]: DEBUG nova.compute.manager [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1128.726395] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1128.727132] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-01d6261a-9226-4506-a1eb-ba917d707950 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1128.733896] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1128.734154] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-bc55d9fb-c804-4747-8441-751c4707f17c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1128.736260] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1128.736437] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1128.737384] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-9423a055-2c6b-48ae-a000-5bf9c43c30db {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1128.741947] env[67752]: DEBUG oslo_vmware.api [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Waiting for the task: (returnval){ [ 1128.741947] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52e74cc6-503c-9fd3-ccdc-fbd304d833ea" [ 1128.741947] env[67752]: _type = "Task" [ 1128.741947] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1128.752522] env[67752]: DEBUG oslo_vmware.api [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52e74cc6-503c-9fd3-ccdc-fbd304d833ea, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1128.798799] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1128.799250] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1128.799646] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Deleting the datastore file [datastore2] ff4895f8-0374-4b34-a5e8-d3ba24f840a5 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1128.799933] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-de600009-a8a1-4a93-82c6-4e5ea5f0ff34 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1128.805964] env[67752]: DEBUG oslo_vmware.api [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Waiting for the task: (returnval){ [ 1128.805964] env[67752]: value = "task-3199725" [ 1128.805964] env[67752]: _type = "Task" [ 1128.805964] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1128.813457] env[67752]: DEBUG oslo_vmware.api [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Task: {'id': task-3199725, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1129.252486] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1129.252753] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Creating directory with path [datastore2] vmware_temp/8e940f54-1554-4c56-a8d0-a6cb47e07285/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1129.252931] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-f0241cd3-047f-4811-b478-45449e41a4b6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1129.263812] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Created directory with path [datastore2] vmware_temp/8e940f54-1554-4c56-a8d0-a6cb47e07285/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1129.264017] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Fetch image to [datastore2] vmware_temp/8e940f54-1554-4c56-a8d0-a6cb47e07285/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1129.264198] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/8e940f54-1554-4c56-a8d0-a6cb47e07285/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1129.264925] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ed65a593-ef31-4681-ab6b-c16b6ef7ece0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1129.272818] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4869c5ef-7a97-478e-b48d-7d6436c01a70 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1129.281855] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bcc5d6d8-da94-486a-b3df-6b4c9fe3eb61 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1129.314711] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b34b4396-65d6-45e4-8b12-4c51ca034f96 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1129.321486] env[67752]: DEBUG oslo_vmware.api [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Task: {'id': task-3199725, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.07863} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1129.322909] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1129.323117] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1129.323295] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1129.323467] env[67752]: INFO nova.compute.manager [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1129.325253] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-7cd999a2-dfd6-4920-9171-a52a89260493 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1129.327155] env[67752]: DEBUG nova.compute.claims [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1129.327331] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1129.327544] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1129.359127] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1129.413759] env[67752]: DEBUG oslo_vmware.rw_handles [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/8e940f54-1554-4c56-a8d0-a6cb47e07285/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1129.477221] env[67752]: DEBUG oslo_vmware.rw_handles [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1129.477432] env[67752]: DEBUG oslo_vmware.rw_handles [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Closing write handle for https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/8e940f54-1554-4c56-a8d0-a6cb47e07285/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1129.733288] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-923957e0-c678-4f48-9ed7-6ef85d11708a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1129.741135] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f321b5ef-cf4a-4f21-b218-31a2512f3a96 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1129.770939] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6009f735-fb25-4d32-b707-2c069994982f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1129.777532] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0dfd698-406d-4cc0-bc23-1d3c8cd9339f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1129.790032] env[67752]: DEBUG nova.compute.provider_tree [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1129.798811] env[67752]: DEBUG nova.scheduler.client.report [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1129.815456] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.488s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1129.815967] env[67752]: ERROR nova.compute.manager [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1129.815967] env[67752]: Faults: ['InvalidArgument'] [ 1129.815967] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Traceback (most recent call last): [ 1129.815967] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1129.815967] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] self.driver.spawn(context, instance, image_meta, [ 1129.815967] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1129.815967] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1129.815967] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1129.815967] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] self._fetch_image_if_missing(context, vi) [ 1129.815967] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1129.815967] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] image_cache(vi, tmp_image_ds_loc) [ 1129.815967] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1129.816378] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] vm_util.copy_virtual_disk( [ 1129.816378] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1129.816378] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] session._wait_for_task(vmdk_copy_task) [ 1129.816378] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1129.816378] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] return self.wait_for_task(task_ref) [ 1129.816378] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1129.816378] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] return evt.wait() [ 1129.816378] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1129.816378] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] result = hub.switch() [ 1129.816378] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1129.816378] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] return self.greenlet.switch() [ 1129.816378] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1129.816378] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] self.f(*self.args, **self.kw) [ 1129.816767] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1129.816767] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] raise exceptions.translate_fault(task_info.error) [ 1129.816767] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1129.816767] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Faults: ['InvalidArgument'] [ 1129.816767] env[67752]: ERROR nova.compute.manager [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] [ 1129.816767] env[67752]: DEBUG nova.compute.utils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1129.818011] env[67752]: DEBUG nova.compute.manager [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Build of instance ff4895f8-0374-4b34-a5e8-d3ba24f840a5 was re-scheduled: A specified parameter was not correct: fileType [ 1129.818011] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1129.818392] env[67752]: DEBUG nova.compute.manager [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1129.818566] env[67752]: DEBUG nova.compute.manager [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1129.818723] env[67752]: DEBUG nova.compute.manager [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1129.818909] env[67752]: DEBUG nova.network.neutron [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1130.497188] env[67752]: DEBUG nova.network.neutron [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1130.510527] env[67752]: INFO nova.compute.manager [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Took 0.69 seconds to deallocate network for instance. [ 1130.628965] env[67752]: INFO nova.scheduler.client.report [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Deleted allocations for instance ff4895f8-0374-4b34-a5e8-d3ba24f840a5 [ 1130.651773] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ccd1f778-e17f-41e6-adfe-e9eca1730ace tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "ff4895f8-0374-4b34-a5e8-d3ba24f840a5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 520.818s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1130.652959] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9b490ea7-b680-4746-bf26-c85ace342ec3 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "ff4895f8-0374-4b34-a5e8-d3ba24f840a5" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 321.563s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1130.653242] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9b490ea7-b680-4746-bf26-c85ace342ec3 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Acquiring lock "ff4895f8-0374-4b34-a5e8-d3ba24f840a5-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1130.653782] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9b490ea7-b680-4746-bf26-c85ace342ec3 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "ff4895f8-0374-4b34-a5e8-d3ba24f840a5-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1130.653962] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9b490ea7-b680-4746-bf26-c85ace342ec3 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "ff4895f8-0374-4b34-a5e8-d3ba24f840a5-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1130.656111] env[67752]: INFO nova.compute.manager [None req-9b490ea7-b680-4746-bf26-c85ace342ec3 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Terminating instance [ 1130.657861] env[67752]: DEBUG nova.compute.manager [None req-9b490ea7-b680-4746-bf26-c85ace342ec3 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1130.658122] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-9b490ea7-b680-4746-bf26-c85ace342ec3 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1130.658550] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-98633a67-b321-4c54-a800-2447963bc758 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1130.671421] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4cd1fbbd-a8be-45b0-92c1-b7d56e8ef727 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1130.679812] env[67752]: DEBUG nova.compute.manager [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1130.703039] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-9b490ea7-b680-4746-bf26-c85ace342ec3 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ff4895f8-0374-4b34-a5e8-d3ba24f840a5 could not be found. [ 1130.703039] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-9b490ea7-b680-4746-bf26-c85ace342ec3 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1130.703039] env[67752]: INFO nova.compute.manager [None req-9b490ea7-b680-4746-bf26-c85ace342ec3 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1130.703039] env[67752]: DEBUG oslo.service.loopingcall [None req-9b490ea7-b680-4746-bf26-c85ace342ec3 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1130.703411] env[67752]: DEBUG nova.compute.manager [-] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1130.703411] env[67752]: DEBUG nova.network.neutron [-] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1130.733877] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1130.734149] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1130.735812] env[67752]: INFO nova.compute.claims [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1130.738802] env[67752]: DEBUG nova.network.neutron [-] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1130.747900] env[67752]: INFO nova.compute.manager [-] [instance: ff4895f8-0374-4b34-a5e8-d3ba24f840a5] Took 0.04 seconds to deallocate network for instance. [ 1130.854519] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9b490ea7-b680-4746-bf26-c85ace342ec3 tempest-ListImageFiltersTestJSON-672772346 tempest-ListImageFiltersTestJSON-672772346-project-member] Lock "ff4895f8-0374-4b34-a5e8-d3ba24f840a5" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.201s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1131.096844] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5398f940-83f7-40c7-bd44-f0ba968160c0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1131.104172] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cabc15ad-cd25-4852-8995-2384b2c206cb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1131.132925] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2226feb3-ab2f-44ed-a336-1ecacf36d275 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1131.139329] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f423ca75-559c-4ea7-bc18-ba43a6180d2f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1131.151880] env[67752]: DEBUG nova.compute.provider_tree [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1131.160568] env[67752]: DEBUG nova.scheduler.client.report [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1131.174188] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.440s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1131.174642] env[67752]: DEBUG nova.compute.manager [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1131.206715] env[67752]: DEBUG nova.compute.utils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1131.208367] env[67752]: DEBUG nova.compute.manager [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1131.208567] env[67752]: DEBUG nova.network.neutron [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1131.218891] env[67752]: DEBUG nova.compute.manager [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1131.277336] env[67752]: DEBUG nova.policy [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '33981823d3944c1ea4b8dd3bc816b58f', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '623ecf0022e946a7ae2c085680601380', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1131.294087] env[67752]: DEBUG nova.compute.manager [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1131.319752] env[67752]: DEBUG nova.virt.hardware [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1131.320401] env[67752]: DEBUG nova.virt.hardware [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1131.320401] env[67752]: DEBUG nova.virt.hardware [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1131.320401] env[67752]: DEBUG nova.virt.hardware [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1131.320558] env[67752]: DEBUG nova.virt.hardware [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1131.320650] env[67752]: DEBUG nova.virt.hardware [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1131.320861] env[67752]: DEBUG nova.virt.hardware [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1131.321021] env[67752]: DEBUG nova.virt.hardware [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1131.321434] env[67752]: DEBUG nova.virt.hardware [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1131.321434] env[67752]: DEBUG nova.virt.hardware [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1131.321577] env[67752]: DEBUG nova.virt.hardware [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1131.322366] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1cbb1766-fd62-4a10-97a1-43498e40f827 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1131.330008] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56b80c9c-1f37-457d-90e6-2af792966b2c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1131.818260] env[67752]: DEBUG nova.network.neutron [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Successfully created port: 374269c8-8c17-4554-91b8-b030716f88fc {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1132.773052] env[67752]: DEBUG nova.network.neutron [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Successfully updated port: 374269c8-8c17-4554-91b8-b030716f88fc {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1132.783520] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquiring lock "refresh_cache-f6499ee5-cb24-44e1-9fe7-c19feeb732eb" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1132.783679] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquired lock "refresh_cache-f6499ee5-cb24-44e1-9fe7-c19feeb732eb" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1132.783833] env[67752]: DEBUG nova.network.neutron [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1132.854949] env[67752]: DEBUG nova.network.neutron [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1133.028565] env[67752]: DEBUG nova.compute.manager [req-d38e964f-c219-4d83-ac40-d65b1bc41f8c req-9a70f797-3889-4961-a094-a43473283007 service nova] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Received event network-vif-plugged-374269c8-8c17-4554-91b8-b030716f88fc {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1133.028809] env[67752]: DEBUG oslo_concurrency.lockutils [req-d38e964f-c219-4d83-ac40-d65b1bc41f8c req-9a70f797-3889-4961-a094-a43473283007 service nova] Acquiring lock "f6499ee5-cb24-44e1-9fe7-c19feeb732eb-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1133.029276] env[67752]: DEBUG oslo_concurrency.lockutils [req-d38e964f-c219-4d83-ac40-d65b1bc41f8c req-9a70f797-3889-4961-a094-a43473283007 service nova] Lock "f6499ee5-cb24-44e1-9fe7-c19feeb732eb-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1133.029492] env[67752]: DEBUG oslo_concurrency.lockutils [req-d38e964f-c219-4d83-ac40-d65b1bc41f8c req-9a70f797-3889-4961-a094-a43473283007 service nova] Lock "f6499ee5-cb24-44e1-9fe7-c19feeb732eb-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1133.029771] env[67752]: DEBUG nova.compute.manager [req-d38e964f-c219-4d83-ac40-d65b1bc41f8c req-9a70f797-3889-4961-a094-a43473283007 service nova] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] No waiting events found dispatching network-vif-plugged-374269c8-8c17-4554-91b8-b030716f88fc {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1133.029957] env[67752]: WARNING nova.compute.manager [req-d38e964f-c219-4d83-ac40-d65b1bc41f8c req-9a70f797-3889-4961-a094-a43473283007 service nova] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Received unexpected event network-vif-plugged-374269c8-8c17-4554-91b8-b030716f88fc for instance with vm_state building and task_state spawning. [ 1133.030046] env[67752]: DEBUG nova.compute.manager [req-d38e964f-c219-4d83-ac40-d65b1bc41f8c req-9a70f797-3889-4961-a094-a43473283007 service nova] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Received event network-changed-374269c8-8c17-4554-91b8-b030716f88fc {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1133.030217] env[67752]: DEBUG nova.compute.manager [req-d38e964f-c219-4d83-ac40-d65b1bc41f8c req-9a70f797-3889-4961-a094-a43473283007 service nova] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Refreshing instance network info cache due to event network-changed-374269c8-8c17-4554-91b8-b030716f88fc. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1133.030390] env[67752]: DEBUG oslo_concurrency.lockutils [req-d38e964f-c219-4d83-ac40-d65b1bc41f8c req-9a70f797-3889-4961-a094-a43473283007 service nova] Acquiring lock "refresh_cache-f6499ee5-cb24-44e1-9fe7-c19feeb732eb" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1133.146205] env[67752]: DEBUG nova.network.neutron [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Updating instance_info_cache with network_info: [{"id": "374269c8-8c17-4554-91b8-b030716f88fc", "address": "fa:16:3e:9a:25:d7", "network": {"id": "17c092d8-0dd2-45ff-8347-48693bf6460a", "bridge": "br-int", "label": "tempest-VolumesAdminNegativeTest-733494480-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "623ecf0022e946a7ae2c085680601380", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f2c424c9-6446-4b2a-af8c-4d9c29117c39", "external-id": "nsx-vlan-transportzone-437", "segmentation_id": 437, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap374269c8-8c", "ovs_interfaceid": "374269c8-8c17-4554-91b8-b030716f88fc", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1133.159894] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Releasing lock "refresh_cache-f6499ee5-cb24-44e1-9fe7-c19feeb732eb" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1133.160423] env[67752]: DEBUG nova.compute.manager [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Instance network_info: |[{"id": "374269c8-8c17-4554-91b8-b030716f88fc", "address": "fa:16:3e:9a:25:d7", "network": {"id": "17c092d8-0dd2-45ff-8347-48693bf6460a", "bridge": "br-int", "label": "tempest-VolumesAdminNegativeTest-733494480-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "623ecf0022e946a7ae2c085680601380", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f2c424c9-6446-4b2a-af8c-4d9c29117c39", "external-id": "nsx-vlan-transportzone-437", "segmentation_id": 437, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap374269c8-8c", "ovs_interfaceid": "374269c8-8c17-4554-91b8-b030716f88fc", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1133.160580] env[67752]: DEBUG oslo_concurrency.lockutils [req-d38e964f-c219-4d83-ac40-d65b1bc41f8c req-9a70f797-3889-4961-a094-a43473283007 service nova] Acquired lock "refresh_cache-f6499ee5-cb24-44e1-9fe7-c19feeb732eb" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1133.160746] env[67752]: DEBUG nova.network.neutron [req-d38e964f-c219-4d83-ac40-d65b1bc41f8c req-9a70f797-3889-4961-a094-a43473283007 service nova] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Refreshing network info cache for port 374269c8-8c17-4554-91b8-b030716f88fc {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1133.162106] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:9a:25:d7', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'f2c424c9-6446-4b2a-af8c-4d9c29117c39', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '374269c8-8c17-4554-91b8-b030716f88fc', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1133.170013] env[67752]: DEBUG oslo.service.loopingcall [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1133.170591] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1133.170843] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-eb887f51-4a62-4826-9631-0e3ebc8c2624 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1133.192965] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1133.192965] env[67752]: value = "task-3199726" [ 1133.192965] env[67752]: _type = "Task" [ 1133.192965] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1133.201275] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199726, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1133.334502] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b2e6be7b-51f6-480d-b71c-50c2efb458f3 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquiring lock "f6499ee5-cb24-44e1-9fe7-c19feeb732eb" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1133.704990] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199726, 'name': CreateVM_Task, 'duration_secs': 0.304243} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1133.705184] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1133.705822] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1133.705990] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1133.706365] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1133.706623] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c7076b8c-e290-4ec8-8bcc-e42af52cfab1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1133.711879] env[67752]: DEBUG oslo_vmware.api [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Waiting for the task: (returnval){ [ 1133.711879] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52999555-5ab9-421a-4ac3-4b4f97e6b039" [ 1133.711879] env[67752]: _type = "Task" [ 1133.711879] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1133.719781] env[67752]: DEBUG oslo_vmware.api [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52999555-5ab9-421a-4ac3-4b4f97e6b039, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1133.818236] env[67752]: DEBUG nova.network.neutron [req-d38e964f-c219-4d83-ac40-d65b1bc41f8c req-9a70f797-3889-4961-a094-a43473283007 service nova] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Updated VIF entry in instance network info cache for port 374269c8-8c17-4554-91b8-b030716f88fc. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1133.818709] env[67752]: DEBUG nova.network.neutron [req-d38e964f-c219-4d83-ac40-d65b1bc41f8c req-9a70f797-3889-4961-a094-a43473283007 service nova] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Updating instance_info_cache with network_info: [{"id": "374269c8-8c17-4554-91b8-b030716f88fc", "address": "fa:16:3e:9a:25:d7", "network": {"id": "17c092d8-0dd2-45ff-8347-48693bf6460a", "bridge": "br-int", "label": "tempest-VolumesAdminNegativeTest-733494480-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "623ecf0022e946a7ae2c085680601380", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "f2c424c9-6446-4b2a-af8c-4d9c29117c39", "external-id": "nsx-vlan-transportzone-437", "segmentation_id": 437, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap374269c8-8c", "ovs_interfaceid": "374269c8-8c17-4554-91b8-b030716f88fc", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1133.829289] env[67752]: DEBUG oslo_concurrency.lockutils [req-d38e964f-c219-4d83-ac40-d65b1bc41f8c req-9a70f797-3889-4961-a094-a43473283007 service nova] Releasing lock "refresh_cache-f6499ee5-cb24-44e1-9fe7-c19feeb732eb" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1134.223685] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1134.223955] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1134.224184] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1137.630637] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1138.635268] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1138.635580] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Cleaning up deleted instances with incomplete migration {{(pid=67752) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11257}} [ 1139.645019] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1140.634988] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1140.635253] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1140.635338] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Cleaning up deleted instances {{(pid=67752) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11219}} [ 1140.647192] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] There are 0 instances to clean {{(pid=67752) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11228}} [ 1141.643069] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1141.667260] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1141.667260] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 1141.667260] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 1141.687191] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1141.687322] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1141.687413] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1141.687546] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1141.687671] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1141.687794] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1141.687917] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1141.688054] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1141.688179] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1141.688300] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1141.688420] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 1141.688920] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1141.689097] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 1142.635291] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1142.635623] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1143.634410] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1144.643655] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1144.643983] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1144.655436] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1144.655678] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1144.655859] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1144.656033] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1144.657236] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-873971ba-6fe6-41a1-b1e7-cebcd1b5e252 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1144.666106] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7151f54b-7d02-4075-9b1f-c66c5a9bce88 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1144.681017] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f9f31f79-f610-478c-b63d-41aef2aaea80 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1144.687270] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ce7b599-cb11-485b-8808-c29d6e0d9e27 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1144.716712] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181012MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1144.716880] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1144.717093] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1144.850094] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ca18af6b-6b80-44f3-bd8b-f61e2596acd7 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1144.850094] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 02942b29-2343-441e-9e2b-aca2af57c849 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1144.850094] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ebd6c719-32ae-410c-a444-db7ad3f8a9c5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1144.850094] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1144.850325] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 411bafb7-ef72-4529-b3f3-0b4955f23788 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1144.850325] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ada70883-9bef-496e-a65f-ca84f6ee6211 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1144.850325] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1144.850425] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b271cd73-a148-48d6-bed9-6a99512457bd actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1144.850465] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1144.850572] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f6499ee5-cb24-44e1-9fe7-c19feeb732eb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1144.862526] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e7d86d5f-f92d-4891-b490-8d0fe37537fb has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1144.872828] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c7081273-b937-4b9f-9369-79162ca200d1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1144.882039] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d3940c29-852a-427b-9027-aa4080150724 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1144.891711] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9065b97a-05a3-4b62-8f4a-418d5b0b209a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1144.900573] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e2fc8dbe-6458-4f5a-9aef-c44ea4d61b08 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1144.909401] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 6da6a17c-b133-4d3c-8b96-6317760a7b3b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1144.918538] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9f409285-631b-48f9-8b80-a3179c09a237 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1144.927593] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3b3fa52f-9512-4de3-8029-5cbafed0b1ae has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1144.936537] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 7c8fc8d3-1f51-4793-8954-93451ebffa74 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1144.945365] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1144.954342] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance a8bd310e-530a-46ab-add7-1b827ea5f399 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1144.964396] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ded2bdfd-71c9-4af4-b595-98ac91fc8203 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1144.973561] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3d423978-c9cf-476f-9df8-d78206dff460 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1144.982432] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 7c9546af-a9b8-4808-b15e-2ff9d2c12df7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1144.990892] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 78e8ccdd-e5ed-4b87-af85-498ca755561b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1144.999097] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ffa5bab7-fce4-4fa7-9e43-c4fc35ee6c45 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1145.007861] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 2573ebf9-e18d-4b07-bde6-459a2466fe0d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1145.016711] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9432cd04-c2df-428b-b31a-93d3ec0c9d1f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1145.016964] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1145.017145] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1145.301443] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c392e8bb-b9b3-49fb-887b-02fec0929ec2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1145.309332] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-02afbd63-2201-4a54-af61-9157bb40c49c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1145.338169] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4ab9097-84b6-4a7f-b7b1-09416a9e627a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1145.345092] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d6f652e-1c6e-4af6-bf40-49a7de385ee6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1145.357912] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1145.366913] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1145.380431] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1145.380629] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.664s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1177.110976] env[67752]: WARNING oslo_vmware.rw_handles [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1177.110976] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1177.110976] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1177.110976] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1177.110976] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1177.110976] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1177.110976] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1177.110976] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1177.110976] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1177.110976] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1177.110976] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1177.110976] env[67752]: ERROR oslo_vmware.rw_handles [ 1177.111682] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/8e940f54-1554-4c56-a8d0-a6cb47e07285/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1177.113446] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1177.113725] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Copying Virtual Disk [datastore2] vmware_temp/8e940f54-1554-4c56-a8d0-a6cb47e07285/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/8e940f54-1554-4c56-a8d0-a6cb47e07285/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1177.114033] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-753dd8cf-cc34-4802-8e32-5104439477ed {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1177.123156] env[67752]: DEBUG oslo_vmware.api [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Waiting for the task: (returnval){ [ 1177.123156] env[67752]: value = "task-3199727" [ 1177.123156] env[67752]: _type = "Task" [ 1177.123156] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1177.132370] env[67752]: DEBUG oslo_vmware.api [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Task: {'id': task-3199727, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1177.634126] env[67752]: DEBUG oslo_vmware.exceptions [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1177.634428] env[67752]: DEBUG oslo_concurrency.lockutils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1177.634991] env[67752]: ERROR nova.compute.manager [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1177.634991] env[67752]: Faults: ['InvalidArgument'] [ 1177.634991] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Traceback (most recent call last): [ 1177.634991] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1177.634991] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] yield resources [ 1177.634991] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1177.634991] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] self.driver.spawn(context, instance, image_meta, [ 1177.634991] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1177.634991] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1177.634991] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1177.634991] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] self._fetch_image_if_missing(context, vi) [ 1177.634991] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1177.635435] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] image_cache(vi, tmp_image_ds_loc) [ 1177.635435] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1177.635435] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] vm_util.copy_virtual_disk( [ 1177.635435] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1177.635435] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] session._wait_for_task(vmdk_copy_task) [ 1177.635435] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1177.635435] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] return self.wait_for_task(task_ref) [ 1177.635435] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1177.635435] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] return evt.wait() [ 1177.635435] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1177.635435] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] result = hub.switch() [ 1177.635435] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1177.635435] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] return self.greenlet.switch() [ 1177.635852] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1177.635852] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] self.f(*self.args, **self.kw) [ 1177.635852] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1177.635852] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] raise exceptions.translate_fault(task_info.error) [ 1177.635852] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1177.635852] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Faults: ['InvalidArgument'] [ 1177.635852] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] [ 1177.635852] env[67752]: INFO nova.compute.manager [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Terminating instance [ 1177.636982] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1177.637244] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1177.637883] env[67752]: DEBUG nova.compute.manager [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1177.638108] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1177.638345] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-d49d8c00-6b80-4009-ae87-3fdcf5f99a4d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1177.640869] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a48d732a-a64f-402a-a8d4-0d7f2089afa5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1177.647373] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1177.647599] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-836a2c49-da33-4022-82f6-9861d65388c8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1177.649749] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1177.649921] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1177.650837] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-d612db43-0067-48f0-b3a8-b935a8d6cb95 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1177.655553] env[67752]: DEBUG oslo_vmware.api [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Waiting for the task: (returnval){ [ 1177.655553] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52898922-a3cf-869a-acfb-b162ee09968d" [ 1177.655553] env[67752]: _type = "Task" [ 1177.655553] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1177.662494] env[67752]: DEBUG oslo_vmware.api [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52898922-a3cf-869a-acfb-b162ee09968d, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1177.720324] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1177.721030] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1177.721030] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Deleting the datastore file [datastore2] ca18af6b-6b80-44f3-bd8b-f61e2596acd7 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1177.721182] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-77486aa0-2119-40f2-8c13-d773e5f18442 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1177.727431] env[67752]: DEBUG oslo_vmware.api [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Waiting for the task: (returnval){ [ 1177.727431] env[67752]: value = "task-3199729" [ 1177.727431] env[67752]: _type = "Task" [ 1177.727431] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1177.734680] env[67752]: DEBUG oslo_vmware.api [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Task: {'id': task-3199729, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1178.165693] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1178.166018] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Creating directory with path [datastore2] vmware_temp/7ddfcef8-18be-4e27-bad8-108a6f0e17fa/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1178.166179] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-a2ce7cdc-8fe9-422b-8bd8-3e109985173c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1178.176779] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Created directory with path [datastore2] vmware_temp/7ddfcef8-18be-4e27-bad8-108a6f0e17fa/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1178.176965] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Fetch image to [datastore2] vmware_temp/7ddfcef8-18be-4e27-bad8-108a6f0e17fa/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1178.177167] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/7ddfcef8-18be-4e27-bad8-108a6f0e17fa/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1178.177869] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d85ae16-58ee-4f2a-b548-87b832021ad8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1178.184434] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-940c1c03-f5f7-4686-a823-4bd8898af47e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1178.192997] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c880a017-68e3-44fd-b2cc-6645ac2b8123 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1178.222641] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-157059c5-b3be-45c3-9b0e-9ecba26fc805 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1178.230685] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-d717e98c-3977-44ca-a72a-d50d8245b636 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1178.237598] env[67752]: DEBUG oslo_vmware.api [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Task: {'id': task-3199729, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.082782} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1178.237819] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1178.237998] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1178.238186] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1178.238359] env[67752]: INFO nova.compute.manager [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1178.240438] env[67752]: DEBUG nova.compute.claims [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1178.240608] env[67752]: DEBUG oslo_concurrency.lockutils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1178.240815] env[67752]: DEBUG oslo_concurrency.lockutils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1178.253810] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1178.306793] env[67752]: DEBUG oslo_vmware.rw_handles [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/7ddfcef8-18be-4e27-bad8-108a6f0e17fa/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1178.370419] env[67752]: DEBUG oslo_vmware.rw_handles [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1178.370610] env[67752]: DEBUG oslo_vmware.rw_handles [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/7ddfcef8-18be-4e27-bad8-108a6f0e17fa/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1178.622011] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f1d907f9-288f-4cd0-8826-fb5f2ee4b18f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1178.629711] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-df61d2ed-ffaf-4127-9875-c4c0666c6e7b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1178.659023] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-60517412-c08c-4c13-a8de-233630fb5daa {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1178.665982] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e5e0924-4a4d-4e29-af9e-5cda9a45372b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1178.678674] env[67752]: DEBUG nova.compute.provider_tree [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1178.687439] env[67752]: DEBUG nova.scheduler.client.report [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1178.701176] env[67752]: DEBUG oslo_concurrency.lockutils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.460s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1178.701759] env[67752]: ERROR nova.compute.manager [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1178.701759] env[67752]: Faults: ['InvalidArgument'] [ 1178.701759] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Traceback (most recent call last): [ 1178.701759] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1178.701759] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] self.driver.spawn(context, instance, image_meta, [ 1178.701759] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1178.701759] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1178.701759] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1178.701759] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] self._fetch_image_if_missing(context, vi) [ 1178.701759] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1178.701759] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] image_cache(vi, tmp_image_ds_loc) [ 1178.701759] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1178.702110] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] vm_util.copy_virtual_disk( [ 1178.702110] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1178.702110] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] session._wait_for_task(vmdk_copy_task) [ 1178.702110] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1178.702110] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] return self.wait_for_task(task_ref) [ 1178.702110] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1178.702110] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] return evt.wait() [ 1178.702110] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1178.702110] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] result = hub.switch() [ 1178.702110] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1178.702110] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] return self.greenlet.switch() [ 1178.702110] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1178.702110] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] self.f(*self.args, **self.kw) [ 1178.702430] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1178.702430] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] raise exceptions.translate_fault(task_info.error) [ 1178.702430] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1178.702430] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Faults: ['InvalidArgument'] [ 1178.702430] env[67752]: ERROR nova.compute.manager [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] [ 1178.702570] env[67752]: DEBUG nova.compute.utils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1178.703830] env[67752]: DEBUG nova.compute.manager [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Build of instance ca18af6b-6b80-44f3-bd8b-f61e2596acd7 was re-scheduled: A specified parameter was not correct: fileType [ 1178.703830] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1178.704277] env[67752]: DEBUG nova.compute.manager [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1178.704444] env[67752]: DEBUG nova.compute.manager [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1178.704615] env[67752]: DEBUG nova.compute.manager [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1178.704776] env[67752]: DEBUG nova.network.neutron [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1179.137416] env[67752]: DEBUG nova.network.neutron [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1179.152774] env[67752]: INFO nova.compute.manager [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Took 0.45 seconds to deallocate network for instance. [ 1179.256352] env[67752]: INFO nova.scheduler.client.report [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Deleted allocations for instance ca18af6b-6b80-44f3-bd8b-f61e2596acd7 [ 1179.282520] env[67752]: DEBUG oslo_concurrency.lockutils [None req-635f8d68-3056-4c73-bcc7-47362bd35cba tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Lock "ca18af6b-6b80-44f3-bd8b-f61e2596acd7" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 568.141s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1179.284098] env[67752]: DEBUG oslo_concurrency.lockutils [None req-23bde4ab-c3d3-4b40-b827-1c117f98a602 tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Lock "ca18af6b-6b80-44f3-bd8b-f61e2596acd7" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 369.460s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1179.284338] env[67752]: DEBUG oslo_concurrency.lockutils [None req-23bde4ab-c3d3-4b40-b827-1c117f98a602 tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Acquiring lock "ca18af6b-6b80-44f3-bd8b-f61e2596acd7-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1179.284560] env[67752]: DEBUG oslo_concurrency.lockutils [None req-23bde4ab-c3d3-4b40-b827-1c117f98a602 tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Lock "ca18af6b-6b80-44f3-bd8b-f61e2596acd7-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1179.284732] env[67752]: DEBUG oslo_concurrency.lockutils [None req-23bde4ab-c3d3-4b40-b827-1c117f98a602 tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Lock "ca18af6b-6b80-44f3-bd8b-f61e2596acd7-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1179.286784] env[67752]: INFO nova.compute.manager [None req-23bde4ab-c3d3-4b40-b827-1c117f98a602 tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Terminating instance [ 1179.288650] env[67752]: DEBUG nova.compute.manager [None req-23bde4ab-c3d3-4b40-b827-1c117f98a602 tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1179.288911] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-23bde4ab-c3d3-4b40-b827-1c117f98a602 tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1179.289424] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-93953f3e-3be9-40fd-8b71-38b26c3df3e6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1179.294700] env[67752]: DEBUG nova.compute.manager [None req-730dd685-a504-4a15-b85b-9b4395944e0f tempest-ServersListShow296Test-1143284993 tempest-ServersListShow296Test-1143284993-project-member] [instance: e7d86d5f-f92d-4891-b490-8d0fe37537fb] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1179.301298] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3bab2f6b-b91e-4a7d-8618-c1743931f649 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1179.320961] env[67752]: DEBUG nova.compute.manager [None req-730dd685-a504-4a15-b85b-9b4395944e0f tempest-ServersListShow296Test-1143284993 tempest-ServersListShow296Test-1143284993-project-member] [instance: e7d86d5f-f92d-4891-b490-8d0fe37537fb] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1179.330200] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-23bde4ab-c3d3-4b40-b827-1c117f98a602 tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ca18af6b-6b80-44f3-bd8b-f61e2596acd7 could not be found. [ 1179.330408] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-23bde4ab-c3d3-4b40-b827-1c117f98a602 tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1179.330584] env[67752]: INFO nova.compute.manager [None req-23bde4ab-c3d3-4b40-b827-1c117f98a602 tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1179.330837] env[67752]: DEBUG oslo.service.loopingcall [None req-23bde4ab-c3d3-4b40-b827-1c117f98a602 tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1179.331293] env[67752]: DEBUG nova.compute.manager [-] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1179.331394] env[67752]: DEBUG nova.network.neutron [-] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1179.348350] env[67752]: DEBUG oslo_concurrency.lockutils [None req-730dd685-a504-4a15-b85b-9b4395944e0f tempest-ServersListShow296Test-1143284993 tempest-ServersListShow296Test-1143284993-project-member] Lock "e7d86d5f-f92d-4891-b490-8d0fe37537fb" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 200.032s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1179.358267] env[67752]: DEBUG nova.compute.manager [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1179.360757] env[67752]: DEBUG nova.network.neutron [-] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1179.371649] env[67752]: INFO nova.compute.manager [-] [instance: ca18af6b-6b80-44f3-bd8b-f61e2596acd7] Took 0.04 seconds to deallocate network for instance. [ 1179.419577] env[67752]: DEBUG oslo_concurrency.lockutils [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1179.420036] env[67752]: DEBUG oslo_concurrency.lockutils [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1179.421283] env[67752]: INFO nova.compute.claims [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1179.470327] env[67752]: DEBUG oslo_concurrency.lockutils [None req-23bde4ab-c3d3-4b40-b827-1c117f98a602 tempest-VolumesAssistedSnapshotsTest-503195977 tempest-VolumesAssistedSnapshotsTest-503195977-project-member] Lock "ca18af6b-6b80-44f3-bd8b-f61e2596acd7" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.186s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1179.594323] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._sync_power_states {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1179.614380] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Getting list of instances from cluster (obj){ [ 1179.614380] env[67752]: value = "domain-c8" [ 1179.614380] env[67752]: _type = "ClusterComputeResource" [ 1179.614380] env[67752]: } {{(pid=67752) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 1179.617710] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f7469d4f-83ae-4ffc-b0a2-7b2ed00e674c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1179.635353] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Got total of 9 instances {{(pid=67752) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 1179.635550] env[67752]: WARNING nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] While synchronizing instance power states, found 10 instances in the database and 9 instances on the hypervisor. [ 1179.635699] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid 02942b29-2343-441e-9e2b-aca2af57c849 {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1179.635889] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid ebd6c719-32ae-410c-a444-db7ad3f8a9c5 {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1179.636057] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9 {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1179.636215] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid 411bafb7-ef72-4529-b3f3-0b4955f23788 {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1179.636376] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid ada70883-9bef-496e-a65f-ca84f6ee6211 {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1179.636526] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70 {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1179.636683] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid b271cd73-a148-48d6-bed9-6a99512457bd {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1179.636828] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9 {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1179.636980] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid f6499ee5-cb24-44e1-9fe7-c19feeb732eb {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1179.637147] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid c7081273-b937-4b9f-9369-79162ca200d1 {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1179.639627] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "02942b29-2343-441e-9e2b-aca2af57c849" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1179.639874] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "ebd6c719-32ae-410c-a444-db7ad3f8a9c5" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1179.640100] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "580c2aa9-160e-40ec-a1e1-0bbaaf60bca9" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1179.640317] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "411bafb7-ef72-4529-b3f3-0b4955f23788" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1179.640845] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "ada70883-9bef-496e-a65f-ca84f6ee6211" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1179.640845] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1179.640946] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "b271cd73-a148-48d6-bed9-6a99512457bd" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1179.641109] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1179.641334] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "f6499ee5-cb24-44e1-9fe7-c19feeb732eb" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1179.641567] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "c7081273-b937-4b9f-9369-79162ca200d1" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1179.765031] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d546a4b0-a845-452c-81ac-ea6d50d335c6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1179.772595] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0795c4c3-b07a-4f44-9dec-23f4d77dab2f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1179.802556] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd182183-efa8-406a-ba6a-3998407c3dba {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1179.812490] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4f1d92f7-b841-4dd6-b132-63da90114d2e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1179.817284] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38b7332b-7322-483b-bb8e-80b5e7b912f1 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquiring lock "c7081273-b937-4b9f-9369-79162ca200d1" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1179.826428] env[67752]: DEBUG nova.compute.provider_tree [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1179.834513] env[67752]: DEBUG nova.scheduler.client.report [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1179.848077] env[67752]: DEBUG oslo_concurrency.lockutils [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.428s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1179.848358] env[67752]: DEBUG nova.compute.manager [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1179.876369] env[67752]: DEBUG nova.compute.claims [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1179.876556] env[67752]: DEBUG oslo_concurrency.lockutils [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1179.876775] env[67752]: DEBUG oslo_concurrency.lockutils [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1180.194516] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42876295-b933-480d-bc96-76a60c56b7e8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1180.203152] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5cae78fa-e341-4915-b189-5ee7208de9b5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1180.233278] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-60bb7f4d-d47e-429e-af58-981a2678b901 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1180.241476] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-738440e0-ad7c-4477-bc47-0d525c8c6e78 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1180.254846] env[67752]: DEBUG nova.compute.provider_tree [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1180.265177] env[67752]: DEBUG nova.scheduler.client.report [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1180.285411] env[67752]: DEBUG oslo_concurrency.lockutils [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.408s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1180.286257] env[67752]: DEBUG nova.compute.utils [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Conflict updating instance c7081273-b937-4b9f-9369-79162ca200d1. Expected: {'task_state': [None]}. Actual: {'task_state': 'deleting'} {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1180.287743] env[67752]: DEBUG nova.compute.manager [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Instance disappeared during build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2504}} [ 1180.287927] env[67752]: DEBUG nova.compute.manager [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1180.288195] env[67752]: DEBUG oslo_concurrency.lockutils [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquiring lock "refresh_cache-c7081273-b937-4b9f-9369-79162ca200d1" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1180.288415] env[67752]: DEBUG oslo_concurrency.lockutils [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquired lock "refresh_cache-c7081273-b937-4b9f-9369-79162ca200d1" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1180.288553] env[67752]: DEBUG nova.network.neutron [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1180.317687] env[67752]: DEBUG nova.network.neutron [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1180.451803] env[67752]: DEBUG nova.network.neutron [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1180.469780] env[67752]: DEBUG oslo_concurrency.lockutils [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Releasing lock "refresh_cache-c7081273-b937-4b9f-9369-79162ca200d1" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1180.470152] env[67752]: DEBUG nova.compute.manager [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1180.470430] env[67752]: DEBUG nova.compute.manager [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1180.470487] env[67752]: DEBUG nova.network.neutron [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1180.491962] env[67752]: DEBUG nova.network.neutron [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1180.499329] env[67752]: DEBUG nova.network.neutron [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1180.508080] env[67752]: INFO nova.compute.manager [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Took 0.04 seconds to deallocate network for instance. [ 1180.582393] env[67752]: INFO nova.scheduler.client.report [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Deleted allocations for instance c7081273-b937-4b9f-9369-79162ca200d1 [ 1180.582606] env[67752]: DEBUG oslo_concurrency.lockutils [None req-72846fbb-f0ab-43c0-8f67-741f97f71c79 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "c7081273-b937-4b9f-9369-79162ca200d1" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 196.906s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1180.583772] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "c7081273-b937-4b9f-9369-79162ca200d1" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 0.942s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1180.584238] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-67dea8f7-168f-4d44-9df5-6686e5fd6e1c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1180.592441] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ad5d96b4-c57a-4fde-9d84-e3f116547d3a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1180.604046] env[67752]: DEBUG nova.compute.manager [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1180.647499] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: c7081273-b937-4b9f-9369-79162ca200d1] During the sync_power process the instance has moved from host None to host cpu-1 [ 1180.648180] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "c7081273-b937-4b9f-9369-79162ca200d1" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.064s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1180.648180] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38b7332b-7322-483b-bb8e-80b5e7b912f1 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "c7081273-b937-4b9f-9369-79162ca200d1" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 0.831s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1180.648180] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38b7332b-7322-483b-bb8e-80b5e7b912f1 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquiring lock "c7081273-b937-4b9f-9369-79162ca200d1-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1180.648390] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38b7332b-7322-483b-bb8e-80b5e7b912f1 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "c7081273-b937-4b9f-9369-79162ca200d1-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1180.648519] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38b7332b-7322-483b-bb8e-80b5e7b912f1 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "c7081273-b937-4b9f-9369-79162ca200d1-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1180.650166] env[67752]: INFO nova.compute.manager [None req-38b7332b-7322-483b-bb8e-80b5e7b912f1 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Terminating instance [ 1180.651992] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1180.652443] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1180.653594] env[67752]: INFO nova.compute.claims [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1180.656045] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38b7332b-7322-483b-bb8e-80b5e7b912f1 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquiring lock "refresh_cache-c7081273-b937-4b9f-9369-79162ca200d1" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1180.656180] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38b7332b-7322-483b-bb8e-80b5e7b912f1 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Acquired lock "refresh_cache-c7081273-b937-4b9f-9369-79162ca200d1" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1180.656348] env[67752]: DEBUG nova.network.neutron [None req-38b7332b-7322-483b-bb8e-80b5e7b912f1 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1180.679949] env[67752]: DEBUG nova.network.neutron [None req-38b7332b-7322-483b-bb8e-80b5e7b912f1 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1180.745375] env[67752]: DEBUG nova.network.neutron [None req-38b7332b-7322-483b-bb8e-80b5e7b912f1 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1180.755583] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38b7332b-7322-483b-bb8e-80b5e7b912f1 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Releasing lock "refresh_cache-c7081273-b937-4b9f-9369-79162ca200d1" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1180.755999] env[67752]: DEBUG nova.compute.manager [None req-38b7332b-7322-483b-bb8e-80b5e7b912f1 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1180.756210] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-38b7332b-7322-483b-bb8e-80b5e7b912f1 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1180.756754] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-7439ed99-a8bc-47a6-ab59-673d58516b73 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1180.766215] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e04b580f-79f1-417e-9ff5-527ca8dcd599 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1180.798871] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-38b7332b-7322-483b-bb8e-80b5e7b912f1 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance c7081273-b937-4b9f-9369-79162ca200d1 could not be found. [ 1180.798871] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-38b7332b-7322-483b-bb8e-80b5e7b912f1 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1180.799024] env[67752]: INFO nova.compute.manager [None req-38b7332b-7322-483b-bb8e-80b5e7b912f1 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1180.800451] env[67752]: DEBUG oslo.service.loopingcall [None req-38b7332b-7322-483b-bb8e-80b5e7b912f1 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1180.800451] env[67752]: DEBUG nova.compute.manager [-] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1180.800451] env[67752]: DEBUG nova.network.neutron [-] [instance: c7081273-b937-4b9f-9369-79162ca200d1] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1180.821575] env[67752]: DEBUG nova.network.neutron [-] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1180.829907] env[67752]: DEBUG nova.network.neutron [-] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1180.839260] env[67752]: INFO nova.compute.manager [-] [instance: c7081273-b937-4b9f-9369-79162ca200d1] Took 0.04 seconds to deallocate network for instance. [ 1180.932118] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38b7332b-7322-483b-bb8e-80b5e7b912f1 tempest-MigrationsAdminTest-1342788076 tempest-MigrationsAdminTest-1342788076-project-member] Lock "c7081273-b937-4b9f-9369-79162ca200d1" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.284s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1181.026731] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e045e972-b7df-4381-8c89-e0f3caf00062 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1181.034387] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7d726144-55f0-4642-9686-50ab0804a688 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1181.062873] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b345f2ac-bdf1-4af6-9b88-a97a9147f3a4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1181.069668] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-416583a4-cd2a-46eb-b7cf-02b61819a252 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1181.082326] env[67752]: DEBUG nova.compute.provider_tree [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1181.090833] env[67752]: DEBUG nova.scheduler.client.report [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1181.104196] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.452s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1181.104653] env[67752]: DEBUG nova.compute.manager [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1181.134354] env[67752]: DEBUG nova.compute.utils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1181.135804] env[67752]: DEBUG nova.compute.manager [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1181.135978] env[67752]: DEBUG nova.network.neutron [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1181.146113] env[67752]: DEBUG nova.compute.manager [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1181.235487] env[67752]: DEBUG nova.compute.manager [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1181.259622] env[67752]: DEBUG nova.policy [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '9e0614e5850f4b1fa2f0f86a1b05730c', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '5680f2c6b86943a69d049f8e18880997', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1181.262685] env[67752]: DEBUG nova.virt.hardware [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1181.262926] env[67752]: DEBUG nova.virt.hardware [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1181.263190] env[67752]: DEBUG nova.virt.hardware [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1181.263255] env[67752]: DEBUG nova.virt.hardware [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1181.263423] env[67752]: DEBUG nova.virt.hardware [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1181.263555] env[67752]: DEBUG nova.virt.hardware [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1181.263760] env[67752]: DEBUG nova.virt.hardware [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1181.263919] env[67752]: DEBUG nova.virt.hardware [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1181.264196] env[67752]: DEBUG nova.virt.hardware [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1181.264271] env[67752]: DEBUG nova.virt.hardware [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1181.264459] env[67752]: DEBUG nova.virt.hardware [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1181.266296] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b844324a-3336-4b4c-aadd-7dfc21659529 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1181.275348] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d5a96c64-07c9-4b7e-810f-bde1e02e1274 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1181.754412] env[67752]: DEBUG nova.network.neutron [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Successfully created port: 1cb951eb-5863-43ba-9fbc-b74cea177234 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1182.969668] env[67752]: DEBUG nova.compute.manager [req-61c1c9c2-f655-4d47-aa39-bcf3f8067b76 req-3b71077c-af3d-4df5-9f99-cb635fe70bf7 service nova] [instance: d3940c29-852a-427b-9027-aa4080150724] Received event network-vif-plugged-1cb951eb-5863-43ba-9fbc-b74cea177234 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1182.969967] env[67752]: DEBUG oslo_concurrency.lockutils [req-61c1c9c2-f655-4d47-aa39-bcf3f8067b76 req-3b71077c-af3d-4df5-9f99-cb635fe70bf7 service nova] Acquiring lock "d3940c29-852a-427b-9027-aa4080150724-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1182.970129] env[67752]: DEBUG oslo_concurrency.lockutils [req-61c1c9c2-f655-4d47-aa39-bcf3f8067b76 req-3b71077c-af3d-4df5-9f99-cb635fe70bf7 service nova] Lock "d3940c29-852a-427b-9027-aa4080150724-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1182.970724] env[67752]: DEBUG oslo_concurrency.lockutils [req-61c1c9c2-f655-4d47-aa39-bcf3f8067b76 req-3b71077c-af3d-4df5-9f99-cb635fe70bf7 service nova] Lock "d3940c29-852a-427b-9027-aa4080150724-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1182.970724] env[67752]: DEBUG nova.compute.manager [req-61c1c9c2-f655-4d47-aa39-bcf3f8067b76 req-3b71077c-af3d-4df5-9f99-cb635fe70bf7 service nova] [instance: d3940c29-852a-427b-9027-aa4080150724] No waiting events found dispatching network-vif-plugged-1cb951eb-5863-43ba-9fbc-b74cea177234 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1182.970724] env[67752]: WARNING nova.compute.manager [req-61c1c9c2-f655-4d47-aa39-bcf3f8067b76 req-3b71077c-af3d-4df5-9f99-cb635fe70bf7 service nova] [instance: d3940c29-852a-427b-9027-aa4080150724] Received unexpected event network-vif-plugged-1cb951eb-5863-43ba-9fbc-b74cea177234 for instance with vm_state building and task_state spawning. [ 1183.036868] env[67752]: DEBUG nova.network.neutron [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Successfully updated port: 1cb951eb-5863-43ba-9fbc-b74cea177234 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1183.053386] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Acquiring lock "refresh_cache-d3940c29-852a-427b-9027-aa4080150724" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1183.053653] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Acquired lock "refresh_cache-d3940c29-852a-427b-9027-aa4080150724" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1183.053738] env[67752]: DEBUG nova.network.neutron [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1183.142990] env[67752]: DEBUG nova.network.neutron [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1183.668934] env[67752]: DEBUG nova.network.neutron [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Updating instance_info_cache with network_info: [{"id": "1cb951eb-5863-43ba-9fbc-b74cea177234", "address": "fa:16:3e:e7:58:13", "network": {"id": "a2699eff-e3c7-4e6c-9318-f65763183aae", "bridge": "br-int", "label": "tempest-InstanceActionsNegativeTestJSON-954826621-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "5680f2c6b86943a69d049f8e18880997", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a1895250-76cc-41f7-b7f8-2e5679494607", "external-id": "nsx-vlan-transportzone-785", "segmentation_id": 785, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap1cb951eb-58", "ovs_interfaceid": "1cb951eb-5863-43ba-9fbc-b74cea177234", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1183.685456] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Releasing lock "refresh_cache-d3940c29-852a-427b-9027-aa4080150724" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1183.685766] env[67752]: DEBUG nova.compute.manager [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Instance network_info: |[{"id": "1cb951eb-5863-43ba-9fbc-b74cea177234", "address": "fa:16:3e:e7:58:13", "network": {"id": "a2699eff-e3c7-4e6c-9318-f65763183aae", "bridge": "br-int", "label": "tempest-InstanceActionsNegativeTestJSON-954826621-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "5680f2c6b86943a69d049f8e18880997", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a1895250-76cc-41f7-b7f8-2e5679494607", "external-id": "nsx-vlan-transportzone-785", "segmentation_id": 785, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap1cb951eb-58", "ovs_interfaceid": "1cb951eb-5863-43ba-9fbc-b74cea177234", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1183.686870] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:e7:58:13', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'a1895250-76cc-41f7-b7f8-2e5679494607', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '1cb951eb-5863-43ba-9fbc-b74cea177234', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1183.694978] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Creating folder: Project (5680f2c6b86943a69d049f8e18880997). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1183.695489] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-157f32a4-8dea-4600-871d-7c2c19e42d24 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1183.706137] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Created folder: Project (5680f2c6b86943a69d049f8e18880997) in parent group-v639722. [ 1183.706365] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Creating folder: Instances. Parent ref: group-v639795. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1183.706559] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-bc6e72cf-4d32-4448-bbde-6db019546349 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1183.715199] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Created folder: Instances in parent group-v639795. [ 1183.715435] env[67752]: DEBUG oslo.service.loopingcall [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1183.715614] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: d3940c29-852a-427b-9027-aa4080150724] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1183.715806] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-9971f32f-cc07-4203-9e5f-2b7c00e4ecc7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1183.734765] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1183.734765] env[67752]: value = "task-3199732" [ 1183.734765] env[67752]: _type = "Task" [ 1183.734765] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1183.743937] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199732, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1184.244539] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199732, 'name': CreateVM_Task, 'duration_secs': 0.413274} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1184.244810] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: d3940c29-852a-427b-9027-aa4080150724] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1184.245446] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1184.245552] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1184.245877] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1184.246136] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-3f1ca6b0-2d96-460e-a98f-160db6a00fab {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1184.250594] env[67752]: DEBUG oslo_vmware.api [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Waiting for the task: (returnval){ [ 1184.250594] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]522d99c6-7366-2d97-a79f-5c7ae3ef6c7a" [ 1184.250594] env[67752]: _type = "Task" [ 1184.250594] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1184.258192] env[67752]: DEBUG oslo_vmware.api [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]522d99c6-7366-2d97-a79f-5c7ae3ef6c7a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1184.762725] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1184.762725] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1184.762725] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1184.993306] env[67752]: DEBUG nova.compute.manager [req-a3e24ac5-ee01-4d93-abf7-bf99f5b52694 req-d5ef97fe-6449-4115-825f-0a67d73dc71c service nova] [instance: d3940c29-852a-427b-9027-aa4080150724] Received event network-changed-1cb951eb-5863-43ba-9fbc-b74cea177234 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1184.993352] env[67752]: DEBUG nova.compute.manager [req-a3e24ac5-ee01-4d93-abf7-bf99f5b52694 req-d5ef97fe-6449-4115-825f-0a67d73dc71c service nova] [instance: d3940c29-852a-427b-9027-aa4080150724] Refreshing instance network info cache due to event network-changed-1cb951eb-5863-43ba-9fbc-b74cea177234. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1184.993569] env[67752]: DEBUG oslo_concurrency.lockutils [req-a3e24ac5-ee01-4d93-abf7-bf99f5b52694 req-d5ef97fe-6449-4115-825f-0a67d73dc71c service nova] Acquiring lock "refresh_cache-d3940c29-852a-427b-9027-aa4080150724" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1184.993727] env[67752]: DEBUG oslo_concurrency.lockutils [req-a3e24ac5-ee01-4d93-abf7-bf99f5b52694 req-d5ef97fe-6449-4115-825f-0a67d73dc71c service nova] Acquired lock "refresh_cache-d3940c29-852a-427b-9027-aa4080150724" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1184.993889] env[67752]: DEBUG nova.network.neutron [req-a3e24ac5-ee01-4d93-abf7-bf99f5b52694 req-d5ef97fe-6449-4115-825f-0a67d73dc71c service nova] [instance: d3940c29-852a-427b-9027-aa4080150724] Refreshing network info cache for port 1cb951eb-5863-43ba-9fbc-b74cea177234 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1185.574996] env[67752]: DEBUG nova.network.neutron [req-a3e24ac5-ee01-4d93-abf7-bf99f5b52694 req-d5ef97fe-6449-4115-825f-0a67d73dc71c service nova] [instance: d3940c29-852a-427b-9027-aa4080150724] Updated VIF entry in instance network info cache for port 1cb951eb-5863-43ba-9fbc-b74cea177234. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1185.575389] env[67752]: DEBUG nova.network.neutron [req-a3e24ac5-ee01-4d93-abf7-bf99f5b52694 req-d5ef97fe-6449-4115-825f-0a67d73dc71c service nova] [instance: d3940c29-852a-427b-9027-aa4080150724] Updating instance_info_cache with network_info: [{"id": "1cb951eb-5863-43ba-9fbc-b74cea177234", "address": "fa:16:3e:e7:58:13", "network": {"id": "a2699eff-e3c7-4e6c-9318-f65763183aae", "bridge": "br-int", "label": "tempest-InstanceActionsNegativeTestJSON-954826621-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "5680f2c6b86943a69d049f8e18880997", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "a1895250-76cc-41f7-b7f8-2e5679494607", "external-id": "nsx-vlan-transportzone-785", "segmentation_id": 785, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap1cb951eb-58", "ovs_interfaceid": "1cb951eb-5863-43ba-9fbc-b74cea177234", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1185.584923] env[67752]: DEBUG oslo_concurrency.lockutils [req-a3e24ac5-ee01-4d93-abf7-bf99f5b52694 req-d5ef97fe-6449-4115-825f-0a67d73dc71c service nova] Releasing lock "refresh_cache-d3940c29-852a-427b-9027-aa4080150724" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1198.677719] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1201.635580] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1201.635973] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 1201.635973] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 1201.659271] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1201.659444] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1201.659569] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1201.659706] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1201.659856] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1201.659984] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1201.660145] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1201.660296] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1201.660424] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1201.660544] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: d3940c29-852a-427b-9027-aa4080150724] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1201.660675] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 1201.661187] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1201.661390] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1201.661527] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 1202.635211] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1202.635454] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1203.634725] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1205.635565] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1205.635565] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1205.647766] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1205.647766] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1205.647766] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1205.649017] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1205.649202] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ba95478-d444-485b-9c77-86fe5c9bb117 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1205.658749] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f812d9d0-9bb0-414e-93f4-7a9e31e9f306 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1205.672535] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4b1c9da0-a7d5-4d9a-8593-7d739e870139 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1205.680026] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44a8bd5a-a31e-4d3f-ae4b-bbf8daebf82d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1205.707077] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180997MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1205.707266] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1205.707464] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1205.780158] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 02942b29-2343-441e-9e2b-aca2af57c849 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1205.780356] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ebd6c719-32ae-410c-a444-db7ad3f8a9c5 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1205.780488] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1205.780610] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 411bafb7-ef72-4529-b3f3-0b4955f23788 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1205.780728] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ada70883-9bef-496e-a65f-ca84f6ee6211 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1205.780846] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1205.780963] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b271cd73-a148-48d6-bed9-6a99512457bd actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1205.781098] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1205.781219] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f6499ee5-cb24-44e1-9fe7-c19feeb732eb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1205.781334] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d3940c29-852a-427b-9027-aa4080150724 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1205.792230] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9065b97a-05a3-4b62-8f4a-418d5b0b209a has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1205.802519] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e2fc8dbe-6458-4f5a-9aef-c44ea4d61b08 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1205.811856] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 6da6a17c-b133-4d3c-8b96-6317760a7b3b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1205.821532] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9f409285-631b-48f9-8b80-a3179c09a237 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1205.830983] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3b3fa52f-9512-4de3-8029-5cbafed0b1ae has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1205.840934] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 7c8fc8d3-1f51-4793-8954-93451ebffa74 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1205.850639] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1205.859624] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance a8bd310e-530a-46ab-add7-1b827ea5f399 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1205.868362] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ded2bdfd-71c9-4af4-b595-98ac91fc8203 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1205.877929] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3d423978-c9cf-476f-9df8-d78206dff460 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1205.887007] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 7c9546af-a9b8-4808-b15e-2ff9d2c12df7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1205.896312] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 78e8ccdd-e5ed-4b87-af85-498ca755561b has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1205.904984] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ffa5bab7-fce4-4fa7-9e43-c4fc35ee6c45 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1205.913895] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 2573ebf9-e18d-4b07-bde6-459a2466fe0d has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1205.923032] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9432cd04-c2df-428b-b31a-93d3ec0c9d1f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1205.923268] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1205.923419] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1206.193329] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7603c9e4-f9d8-4fc2-ba6c-34b9b030adb8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1206.201258] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d0e4650-d0ee-44bd-92ee-387c054ada91 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1206.230115] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81d397c7-95d3-4b38-8cf8-344e65e0151a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1206.236865] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21b2c19a-262c-4704-b212-cb41663cb383 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1206.251887] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1206.260024] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1206.273199] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1206.273380] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.566s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1209.836134] env[67752]: DEBUG oslo_concurrency.lockutils [None req-723fed07-ce60-447e-acf7-4365158dbdc1 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Acquiring lock "d3940c29-852a-427b-9027-aa4080150724" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1220.981786] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Acquiring lock "90e58033-eacd-433a-8d25-40f7d8403588" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1220.981786] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Lock "90e58033-eacd-433a-8d25-40f7d8403588" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1226.341351] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquiring lock "81738fe6-6d50-4dfe-ae86-4c8771837168" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1226.341351] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "81738fe6-6d50-4dfe-ae86-4c8771837168" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1228.235058] env[67752]: WARNING oslo_vmware.rw_handles [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1228.235058] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1228.235058] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1228.235058] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1228.235058] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1228.235058] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1228.235058] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1228.235058] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1228.235058] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1228.235058] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1228.235058] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1228.235058] env[67752]: ERROR oslo_vmware.rw_handles [ 1228.235710] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/7ddfcef8-18be-4e27-bad8-108a6f0e17fa/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1228.237534] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1228.237781] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Copying Virtual Disk [datastore2] vmware_temp/7ddfcef8-18be-4e27-bad8-108a6f0e17fa/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/7ddfcef8-18be-4e27-bad8-108a6f0e17fa/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1228.238086] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-e26dc574-90eb-4082-bc20-b1a5afb5301d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1228.247590] env[67752]: DEBUG oslo_vmware.api [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Waiting for the task: (returnval){ [ 1228.247590] env[67752]: value = "task-3199733" [ 1228.247590] env[67752]: _type = "Task" [ 1228.247590] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1228.255559] env[67752]: DEBUG oslo_vmware.api [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Task: {'id': task-3199733, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1228.759049] env[67752]: DEBUG oslo_vmware.exceptions [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1228.759049] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1228.759293] env[67752]: ERROR nova.compute.manager [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1228.759293] env[67752]: Faults: ['InvalidArgument'] [ 1228.759293] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Traceback (most recent call last): [ 1228.759293] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1228.759293] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] yield resources [ 1228.759293] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1228.759293] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] self.driver.spawn(context, instance, image_meta, [ 1228.759293] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1228.759293] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1228.759293] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1228.759293] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] self._fetch_image_if_missing(context, vi) [ 1228.759293] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1228.759293] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] image_cache(vi, tmp_image_ds_loc) [ 1228.759806] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1228.759806] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] vm_util.copy_virtual_disk( [ 1228.759806] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1228.759806] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] session._wait_for_task(vmdk_copy_task) [ 1228.759806] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1228.759806] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] return self.wait_for_task(task_ref) [ 1228.759806] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1228.759806] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] return evt.wait() [ 1228.759806] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1228.759806] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] result = hub.switch() [ 1228.759806] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1228.759806] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] return self.greenlet.switch() [ 1228.759806] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1228.760234] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] self.f(*self.args, **self.kw) [ 1228.760234] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1228.760234] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] raise exceptions.translate_fault(task_info.error) [ 1228.760234] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1228.760234] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Faults: ['InvalidArgument'] [ 1228.760234] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] [ 1228.760234] env[67752]: INFO nova.compute.manager [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Terminating instance [ 1228.761254] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1228.761475] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1228.762105] env[67752]: DEBUG nova.compute.manager [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1228.762333] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1228.762565] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-cfbc27e9-70c6-4f88-a98d-fb3233a95d70 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1228.764742] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3859b1f4-b66b-4e05-a0ee-d02c2491a80f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1228.771572] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1228.771804] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-a3656d17-9e4a-4275-8431-63a4178c3a92 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1228.773983] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1228.774196] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1228.775107] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-21cc12f8-ba84-4f51-93ad-1df9e0a5f807 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1228.779720] env[67752]: DEBUG oslo_vmware.api [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Waiting for the task: (returnval){ [ 1228.779720] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52a65a7b-8e2c-0682-dc98-f0a5357a7fa4" [ 1228.779720] env[67752]: _type = "Task" [ 1228.779720] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1228.787899] env[67752]: DEBUG oslo_vmware.api [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52a65a7b-8e2c-0682-dc98-f0a5357a7fa4, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1228.838716] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1228.839085] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1228.839300] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Deleting the datastore file [datastore2] 02942b29-2343-441e-9e2b-aca2af57c849 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1228.839576] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-429e545a-2ff5-4552-906c-680839f5fb2e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1228.845525] env[67752]: DEBUG oslo_vmware.api [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Waiting for the task: (returnval){ [ 1228.845525] env[67752]: value = "task-3199735" [ 1228.845525] env[67752]: _type = "Task" [ 1228.845525] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1228.853079] env[67752]: DEBUG oslo_vmware.api [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Task: {'id': task-3199735, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1229.289901] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1229.290296] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Creating directory with path [datastore2] vmware_temp/b0ad68af-452f-4ab3-a356-de6b11e0f2d1/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1229.290409] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-2f475505-3b1a-4723-9095-e81caf2dec4d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1229.301583] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Created directory with path [datastore2] vmware_temp/b0ad68af-452f-4ab3-a356-de6b11e0f2d1/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1229.301768] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Fetch image to [datastore2] vmware_temp/b0ad68af-452f-4ab3-a356-de6b11e0f2d1/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1229.301938] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/b0ad68af-452f-4ab3-a356-de6b11e0f2d1/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1229.302698] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-af8ed9da-cdb1-47e3-9205-ad70c79a6d43 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1229.308992] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56044e09-aa73-42da-9e1a-ab8985491144 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1229.318136] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c16c033b-c32c-46a1-8841-0a045ab5780c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1229.351221] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-947d2bff-6c36-45e5-b580-eba79312fc5b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1229.358914] env[67752]: DEBUG oslo_vmware.api [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Task: {'id': task-3199735, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.076706} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1229.360378] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1229.360584] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1229.360761] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1229.360935] env[67752]: INFO nova.compute.manager [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1229.362748] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-4e20b464-1211-4fe7-b991-24de0072cb1f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1229.365398] env[67752]: DEBUG nova.compute.claims [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1229.365570] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1229.365789] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1229.386819] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1229.690939] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1229.692533] env[67752]: ERROR nova.compute.manager [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Instance failed to spawn: nova.exception.ImageNotAuthorized: Not authorized for image 5e3817d9-8178-4d2c-9c59-eb1eb0833380. [ 1229.692533] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Traceback (most recent call last): [ 1229.692533] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1229.692533] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1229.692533] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1229.692533] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] result = getattr(controller, method)(*args, **kwargs) [ 1229.692533] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1229.692533] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self._get(image_id) [ 1229.692533] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1229.692533] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1229.692533] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1229.693098] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] resp, body = self.http_client.get(url, headers=header) [ 1229.693098] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1229.693098] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self.request(url, 'GET', **kwargs) [ 1229.693098] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1229.693098] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self._handle_response(resp) [ 1229.693098] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1229.693098] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] raise exc.from_response(resp, resp.content) [ 1229.693098] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] glanceclient.exc.HTTPUnauthorized: HTTP 401 Unauthorized: This server could not verify that you are authorized to access the document you requested. Either you supplied the wrong credentials (e.g., bad password), or your browser does not understand how to supply the credentials required. [ 1229.693098] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] [ 1229.693098] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] During handling of the above exception, another exception occurred: [ 1229.693098] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] [ 1229.693098] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Traceback (most recent call last): [ 1229.693765] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1229.693765] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] yield resources [ 1229.693765] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1229.693765] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self.driver.spawn(context, instance, image_meta, [ 1229.693765] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1229.693765] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1229.693765] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1229.693765] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self._fetch_image_if_missing(context, vi) [ 1229.693765] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 637, in _fetch_image_if_missing [ 1229.693765] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] image_fetch(context, vi, tmp_image_ds_loc) [ 1229.693765] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 420, in _fetch_image_as_file [ 1229.693765] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] images.fetch_image( [ 1229.693765] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/virt/vmwareapi/images.py", line 251, in fetch_image [ 1229.694357] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] metadata = IMAGE_API.get(context, image_ref) [ 1229.694357] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 1206, in get [ 1229.694357] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return session.show(context, image_id, [ 1229.694357] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 287, in show [ 1229.694357] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] _reraise_translated_image_exception(image_id) [ 1229.694357] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 1032, in _reraise_translated_image_exception [ 1229.694357] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] raise new_exc.with_traceback(exc_trace) [ 1229.694357] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1229.694357] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1229.694357] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1229.694357] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] result = getattr(controller, method)(*args, **kwargs) [ 1229.694357] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1229.694357] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self._get(image_id) [ 1229.695149] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1229.695149] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1229.695149] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1229.695149] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] resp, body = self.http_client.get(url, headers=header) [ 1229.695149] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1229.695149] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self.request(url, 'GET', **kwargs) [ 1229.695149] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1229.695149] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self._handle_response(resp) [ 1229.695149] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1229.695149] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] raise exc.from_response(resp, resp.content) [ 1229.695149] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] nova.exception.ImageNotAuthorized: Not authorized for image 5e3817d9-8178-4d2c-9c59-eb1eb0833380. [ 1229.695149] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] [ 1229.695649] env[67752]: INFO nova.compute.manager [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Terminating instance [ 1229.695649] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1229.695649] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1229.695649] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-eaeb620c-bc12-434a-9b57-c74456938aff {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1229.697139] env[67752]: DEBUG nova.compute.manager [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1229.697362] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1229.698161] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ea1cbdc2-8d11-4036-b446-4f9066de67c2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1229.709117] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1229.709355] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-d38dbdc3-f326-4545-8512-f47a99ab7241 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1229.712041] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1229.712041] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1229.712688] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-52ec29e3-e32f-4e3b-b07b-5f6e2385379c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1229.719649] env[67752]: DEBUG oslo_vmware.api [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Waiting for the task: (returnval){ [ 1229.719649] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5208a9a3-2783-2ee9-c37c-081316e3164a" [ 1229.719649] env[67752]: _type = "Task" [ 1229.719649] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1229.727167] env[67752]: DEBUG oslo_vmware.api [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5208a9a3-2783-2ee9-c37c-081316e3164a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1229.762325] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-066db518-ac84-4b62-8618-f35fd9e093a3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1229.769149] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1229.769399] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1229.769590] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Deleting the datastore file [datastore2] ebd6c719-32ae-410c-a444-db7ad3f8a9c5 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1229.771400] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-61f1f687-6c1e-45aa-b1d6-4308bfb7ea84 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1229.773931] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-578a88f6-e1f5-496d-87a2-524e9efc6b4c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1229.804488] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c2336df0-c8aa-4c78-a4ef-4cb284e661d9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1229.806922] env[67752]: DEBUG oslo_vmware.api [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Waiting for the task: (returnval){ [ 1229.806922] env[67752]: value = "task-3199737" [ 1229.806922] env[67752]: _type = "Task" [ 1229.806922] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1229.813489] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f04771de-2568-4551-aea6-81b9b423d5f0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1229.819832] env[67752]: DEBUG oslo_vmware.api [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Task: {'id': task-3199737, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1229.829300] env[67752]: DEBUG nova.compute.provider_tree [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1229.839739] env[67752]: DEBUG nova.scheduler.client.report [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1229.857085] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.491s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1229.857396] env[67752]: ERROR nova.compute.manager [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1229.857396] env[67752]: Faults: ['InvalidArgument'] [ 1229.857396] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Traceback (most recent call last): [ 1229.857396] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1229.857396] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] self.driver.spawn(context, instance, image_meta, [ 1229.857396] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1229.857396] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1229.857396] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1229.857396] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] self._fetch_image_if_missing(context, vi) [ 1229.857396] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1229.857396] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] image_cache(vi, tmp_image_ds_loc) [ 1229.857396] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1229.857824] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] vm_util.copy_virtual_disk( [ 1229.857824] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1229.857824] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] session._wait_for_task(vmdk_copy_task) [ 1229.857824] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1229.857824] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] return self.wait_for_task(task_ref) [ 1229.857824] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1229.857824] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] return evt.wait() [ 1229.857824] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1229.857824] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] result = hub.switch() [ 1229.857824] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1229.857824] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] return self.greenlet.switch() [ 1229.857824] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1229.857824] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] self.f(*self.args, **self.kw) [ 1229.858255] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1229.858255] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] raise exceptions.translate_fault(task_info.error) [ 1229.858255] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1229.858255] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Faults: ['InvalidArgument'] [ 1229.858255] env[67752]: ERROR nova.compute.manager [instance: 02942b29-2343-441e-9e2b-aca2af57c849] [ 1229.858255] env[67752]: DEBUG nova.compute.utils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1229.859966] env[67752]: DEBUG nova.compute.manager [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Build of instance 02942b29-2343-441e-9e2b-aca2af57c849 was re-scheduled: A specified parameter was not correct: fileType [ 1229.859966] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1229.860423] env[67752]: DEBUG nova.compute.manager [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1229.860647] env[67752]: DEBUG nova.compute.manager [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1229.860858] env[67752]: DEBUG nova.compute.manager [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1229.861123] env[67752]: DEBUG nova.network.neutron [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1230.230991] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1230.231282] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Creating directory with path [datastore2] vmware_temp/d529c138-219e-47d1-b4e1-213098dbeead/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1230.231523] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-a2e3fd5e-5a24-42a1-9c08-4afbeb238dc1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1230.244466] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Created directory with path [datastore2] vmware_temp/d529c138-219e-47d1-b4e1-213098dbeead/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1230.244758] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Fetch image to [datastore2] vmware_temp/d529c138-219e-47d1-b4e1-213098dbeead/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1230.244897] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/d529c138-219e-47d1-b4e1-213098dbeead/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1230.246027] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c8d35cb0-31f6-4846-ac41-5278ed02186e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1230.254524] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc60b02e-b43e-486d-b31a-fe4a14135217 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1230.264010] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42bd696a-5b18-4b43-9b75-97da08d41f3f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1230.295965] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee839955-a39e-4a63-a4c0-1aa4185ec4a5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1230.302146] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-6ed04aff-1e6f-434c-a656-896cfb09ea00 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1230.318485] env[67752]: DEBUG oslo_vmware.api [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Task: {'id': task-3199737, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.103867} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1230.318772] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1230.318995] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1230.319199] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1230.319426] env[67752]: INFO nova.compute.manager [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Took 0.62 seconds to destroy the instance on the hypervisor. [ 1230.322684] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1230.324847] env[67752]: DEBUG nova.compute.claims [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1230.325082] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1230.325495] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1230.388639] env[67752]: DEBUG oslo_vmware.rw_handles [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/d529c138-219e-47d1-b4e1-213098dbeead/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1230.452763] env[67752]: DEBUG oslo_vmware.rw_handles [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1230.452984] env[67752]: DEBUG oslo_vmware.rw_handles [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Closing write handle for https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/d529c138-219e-47d1-b4e1-213098dbeead/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1230.743014] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7c30a0e-4222-4a4d-afad-9cd0cc9fd49f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1230.749061] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-51f0fc3c-3e83-4022-8343-efc74d891dca {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1230.784949] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-060b6e6d-cf86-4302-80a8-90fd81993519 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1230.793765] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a97ffb8a-52fc-4652-890d-e3bc2baa5289 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1230.807091] env[67752]: DEBUG nova.compute.provider_tree [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1230.817235] env[67752]: DEBUG nova.scheduler.client.report [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1230.833194] env[67752]: DEBUG nova.network.neutron [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1230.836009] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.511s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1230.836998] env[67752]: ERROR nova.compute.manager [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Failed to build and run instance: nova.exception.ImageNotAuthorized: Not authorized for image 5e3817d9-8178-4d2c-9c59-eb1eb0833380. [ 1230.836998] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Traceback (most recent call last): [ 1230.836998] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1230.836998] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1230.836998] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1230.836998] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] result = getattr(controller, method)(*args, **kwargs) [ 1230.836998] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1230.836998] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self._get(image_id) [ 1230.836998] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1230.836998] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1230.836998] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1230.837360] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] resp, body = self.http_client.get(url, headers=header) [ 1230.837360] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1230.837360] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self.request(url, 'GET', **kwargs) [ 1230.837360] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1230.837360] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self._handle_response(resp) [ 1230.837360] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1230.837360] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] raise exc.from_response(resp, resp.content) [ 1230.837360] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] glanceclient.exc.HTTPUnauthorized: HTTP 401 Unauthorized: This server could not verify that you are authorized to access the document you requested. Either you supplied the wrong credentials (e.g., bad password), or your browser does not understand how to supply the credentials required. [ 1230.837360] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] [ 1230.837360] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] During handling of the above exception, another exception occurred: [ 1230.837360] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] [ 1230.837360] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Traceback (most recent call last): [ 1230.837695] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1230.837695] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self.driver.spawn(context, instance, image_meta, [ 1230.837695] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1230.837695] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1230.837695] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1230.837695] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self._fetch_image_if_missing(context, vi) [ 1230.837695] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 637, in _fetch_image_if_missing [ 1230.837695] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] image_fetch(context, vi, tmp_image_ds_loc) [ 1230.837695] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 420, in _fetch_image_as_file [ 1230.837695] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] images.fetch_image( [ 1230.837695] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/virt/vmwareapi/images.py", line 251, in fetch_image [ 1230.837695] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] metadata = IMAGE_API.get(context, image_ref) [ 1230.837695] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 1206, in get [ 1230.838059] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return session.show(context, image_id, [ 1230.838059] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 287, in show [ 1230.838059] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] _reraise_translated_image_exception(image_id) [ 1230.838059] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 1032, in _reraise_translated_image_exception [ 1230.838059] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] raise new_exc.with_traceback(exc_trace) [ 1230.838059] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1230.838059] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1230.838059] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1230.838059] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] result = getattr(controller, method)(*args, **kwargs) [ 1230.838059] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1230.838059] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self._get(image_id) [ 1230.838059] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1230.838059] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1230.838413] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1230.838413] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] resp, body = self.http_client.get(url, headers=header) [ 1230.838413] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1230.838413] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self.request(url, 'GET', **kwargs) [ 1230.838413] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1230.838413] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self._handle_response(resp) [ 1230.838413] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1230.838413] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] raise exc.from_response(resp, resp.content) [ 1230.838413] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] nova.exception.ImageNotAuthorized: Not authorized for image 5e3817d9-8178-4d2c-9c59-eb1eb0833380. [ 1230.838413] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] [ 1230.838710] env[67752]: DEBUG nova.compute.utils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Not authorized for image 5e3817d9-8178-4d2c-9c59-eb1eb0833380. {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1230.840371] env[67752]: DEBUG nova.compute.manager [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Build of instance ebd6c719-32ae-410c-a444-db7ad3f8a9c5 was re-scheduled: Not authorized for image 5e3817d9-8178-4d2c-9c59-eb1eb0833380. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1230.840371] env[67752]: DEBUG nova.compute.manager [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1230.840371] env[67752]: DEBUG nova.compute.manager [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1230.840371] env[67752]: DEBUG nova.compute.manager [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1230.840701] env[67752]: DEBUG nova.network.neutron [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1230.843455] env[67752]: INFO nova.compute.manager [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Took 0.98 seconds to deallocate network for instance. [ 1230.968186] env[67752]: DEBUG neutronclient.v2_0.client [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Error message: {"error": {"code": 401, "title": "Unauthorized", "message": "The request you have made requires authentication."}} {{(pid=67752) _handle_fault_response /opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py:262}} [ 1230.972415] env[67752]: ERROR nova.compute.manager [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Failed to deallocate networks: nova.exception.Unauthorized: Not authorized. [ 1230.972415] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Traceback (most recent call last): [ 1230.972415] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1230.972415] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1230.972415] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1230.972415] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] result = getattr(controller, method)(*args, **kwargs) [ 1230.972415] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1230.972415] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self._get(image_id) [ 1230.972415] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1230.972415] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1230.972415] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1230.972959] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] resp, body = self.http_client.get(url, headers=header) [ 1230.972959] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1230.972959] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self.request(url, 'GET', **kwargs) [ 1230.972959] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1230.972959] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self._handle_response(resp) [ 1230.972959] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1230.972959] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] raise exc.from_response(resp, resp.content) [ 1230.972959] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] glanceclient.exc.HTTPUnauthorized: HTTP 401 Unauthorized: This server could not verify that you are authorized to access the document you requested. Either you supplied the wrong credentials (e.g., bad password), or your browser does not understand how to supply the credentials required. [ 1230.972959] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] [ 1230.972959] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] During handling of the above exception, another exception occurred: [ 1230.972959] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] [ 1230.972959] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Traceback (most recent call last): [ 1230.973364] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1230.973364] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self.driver.spawn(context, instance, image_meta, [ 1230.973364] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1230.973364] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1230.973364] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1230.973364] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self._fetch_image_if_missing(context, vi) [ 1230.973364] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 637, in _fetch_image_if_missing [ 1230.973364] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] image_fetch(context, vi, tmp_image_ds_loc) [ 1230.973364] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 420, in _fetch_image_as_file [ 1230.973364] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] images.fetch_image( [ 1230.973364] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/virt/vmwareapi/images.py", line 251, in fetch_image [ 1230.973364] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] metadata = IMAGE_API.get(context, image_ref) [ 1230.973364] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 1206, in get [ 1230.973763] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return session.show(context, image_id, [ 1230.973763] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 287, in show [ 1230.973763] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] _reraise_translated_image_exception(image_id) [ 1230.973763] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 1032, in _reraise_translated_image_exception [ 1230.973763] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] raise new_exc.with_traceback(exc_trace) [ 1230.973763] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 285, in show [ 1230.973763] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] image = self._client.call(context, 2, 'get', args=(image_id,)) [ 1230.973763] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/image/glance.py", line 191, in call [ 1230.973763] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] result = getattr(controller, method)(*args, **kwargs) [ 1230.973763] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 197, in get [ 1230.973763] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self._get(image_id) [ 1230.973763] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/utils.py", line 649, in inner [ 1230.973763] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return RequestIdProxy(wrapped(*args, **kwargs)) [ 1230.974158] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/v2/images.py", line 190, in _get [ 1230.974158] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] resp, body = self.http_client.get(url, headers=header) [ 1230.974158] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/keystoneauth1/adapter.py", line 393, in get [ 1230.974158] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self.request(url, 'GET', **kwargs) [ 1230.974158] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 380, in request [ 1230.974158] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self._handle_response(resp) [ 1230.974158] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/glanceclient/common/http.py", line 120, in _handle_response [ 1230.974158] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] raise exc.from_response(resp, resp.content) [ 1230.974158] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] nova.exception.ImageNotAuthorized: Not authorized for image 5e3817d9-8178-4d2c-9c59-eb1eb0833380. [ 1230.974158] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] [ 1230.974158] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] During handling of the above exception, another exception occurred: [ 1230.974158] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] [ 1230.974158] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Traceback (most recent call last): [ 1230.974536] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance [ 1230.974536] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self._build_and_run_instance(context, instance, image, [ 1230.974536] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance [ 1230.974536] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] raise exception.RescheduledException( [ 1230.974536] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] nova.exception.RescheduledException: Build of instance ebd6c719-32ae-410c-a444-db7ad3f8a9c5 was re-scheduled: Not authorized for image 5e3817d9-8178-4d2c-9c59-eb1eb0833380. [ 1230.974536] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] [ 1230.974536] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] During handling of the above exception, another exception occurred: [ 1230.974536] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] [ 1230.974536] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Traceback (most recent call last): [ 1230.974536] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1230.974536] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] ret = obj(*args, **kwargs) [ 1230.974536] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1230.974536] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] exception_handler_v20(status_code, error_body) [ 1230.974923] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1230.974923] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] raise client_exc(message=error_message, [ 1230.974923] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1230.974923] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Neutron server returns request_ids: ['req-a7508667-009e-44b8-9673-bd7599741c4f'] [ 1230.974923] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] [ 1230.974923] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] During handling of the above exception, another exception occurred: [ 1230.974923] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] [ 1230.974923] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Traceback (most recent call last): [ 1230.974923] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/compute/manager.py", line 3036, in _cleanup_allocated_networks [ 1230.974923] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self._deallocate_network(context, instance, requested_networks) [ 1230.974923] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/compute/manager.py", line 2282, in _deallocate_network [ 1230.974923] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self.network_api.deallocate_for_instance( [ 1230.974923] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1230.975305] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] data = neutron.list_ports(**search_opts) [ 1230.975305] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1230.975305] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] ret = obj(*args, **kwargs) [ 1230.975305] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1230.975305] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self.list('ports', self.ports_path, retrieve_all, [ 1230.975305] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1230.975305] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] ret = obj(*args, **kwargs) [ 1230.975305] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1230.975305] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] for r in self._pagination(collection, path, **params): [ 1230.975305] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1230.975305] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] res = self.get(path, params=params) [ 1230.975305] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1230.975305] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] ret = obj(*args, **kwargs) [ 1230.975681] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1230.975681] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self.retry_request("GET", action, body=body, [ 1230.975681] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1230.975681] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] ret = obj(*args, **kwargs) [ 1230.975681] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1230.975681] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self.do_request(method, action, body=body, [ 1230.975681] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1230.975681] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] ret = obj(*args, **kwargs) [ 1230.975681] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1230.975681] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self._handle_fault_response(status_code, replybody, resp) [ 1230.975681] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/network/neutron.py", line 204, in wrapper [ 1230.975681] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] raise exception.Unauthorized() [ 1230.975681] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] nova.exception.Unauthorized: Not authorized. [ 1230.976040] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] [ 1230.976040] env[67752]: INFO nova.scheduler.client.report [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Deleted allocations for instance 02942b29-2343-441e-9e2b-aca2af57c849 [ 1231.011473] env[67752]: DEBUG oslo_concurrency.lockutils [None req-38ae1eef-5c7c-4dd3-b23e-450502795001 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Lock "02942b29-2343-441e-9e2b-aca2af57c849" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 614.618s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1231.013027] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4afda383-bd2a-490e-9e57-c8e10d9bd133 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Lock "02942b29-2343-441e-9e2b-aca2af57c849" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 417.365s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1231.013027] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4afda383-bd2a-490e-9e57-c8e10d9bd133 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Acquiring lock "02942b29-2343-441e-9e2b-aca2af57c849-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1231.013243] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4afda383-bd2a-490e-9e57-c8e10d9bd133 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Lock "02942b29-2343-441e-9e2b-aca2af57c849-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1231.013383] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4afda383-bd2a-490e-9e57-c8e10d9bd133 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Lock "02942b29-2343-441e-9e2b-aca2af57c849-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1231.016537] env[67752]: INFO nova.compute.manager [None req-4afda383-bd2a-490e-9e57-c8e10d9bd133 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Terminating instance [ 1231.018761] env[67752]: DEBUG nova.compute.manager [None req-4afda383-bd2a-490e-9e57-c8e10d9bd133 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1231.018946] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-4afda383-bd2a-490e-9e57-c8e10d9bd133 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1231.022398] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-c63c8b58-0bca-4274-b9db-190a05d6c940 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1231.034102] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0667691-2786-4e22-88e1-279ea93deb8d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1231.047281] env[67752]: DEBUG nova.compute.manager [None req-db8f6f02-176a-4689-a251-166bd122b16f tempest-ListServersNegativeTestJSON-684094107 tempest-ListServersNegativeTestJSON-684094107-project-member] [instance: 9065b97a-05a3-4b62-8f4a-418d5b0b209a] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1231.051308] env[67752]: INFO nova.scheduler.client.report [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Deleted allocations for instance ebd6c719-32ae-410c-a444-db7ad3f8a9c5 [ 1231.078028] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-4afda383-bd2a-490e-9e57-c8e10d9bd133 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 02942b29-2343-441e-9e2b-aca2af57c849 could not be found. [ 1231.078028] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-4afda383-bd2a-490e-9e57-c8e10d9bd133 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1231.078028] env[67752]: INFO nova.compute.manager [None req-4afda383-bd2a-490e-9e57-c8e10d9bd133 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Took 0.06 seconds to destroy the instance on the hypervisor. [ 1231.078564] env[67752]: DEBUG oslo.service.loopingcall [None req-4afda383-bd2a-490e-9e57-c8e10d9bd133 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1231.078664] env[67752]: DEBUG nova.compute.manager [-] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1231.078738] env[67752]: DEBUG nova.network.neutron [-] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1231.082980] env[67752]: DEBUG nova.compute.manager [None req-db8f6f02-176a-4689-a251-166bd122b16f tempest-ListServersNegativeTestJSON-684094107 tempest-ListServersNegativeTestJSON-684094107-project-member] [instance: 9065b97a-05a3-4b62-8f4a-418d5b0b209a] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1231.084630] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3cb9ea2-4860-4910-9b60-0c80aae47867 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Lock "ebd6c719-32ae-410c-a444-db7ad3f8a9c5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 608.427s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1231.085973] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a62a8de1-5e3c-479b-b435-c99583f74b98 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Lock "ebd6c719-32ae-410c-a444-db7ad3f8a9c5" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 411.483s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1231.087974] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a62a8de1-5e3c-479b-b435-c99583f74b98 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Acquiring lock "ebd6c719-32ae-410c-a444-db7ad3f8a9c5-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1231.087974] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a62a8de1-5e3c-479b-b435-c99583f74b98 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Lock "ebd6c719-32ae-410c-a444-db7ad3f8a9c5-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1231.087974] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a62a8de1-5e3c-479b-b435-c99583f74b98 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Lock "ebd6c719-32ae-410c-a444-db7ad3f8a9c5-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1231.088667] env[67752]: INFO nova.compute.manager [None req-a62a8de1-5e3c-479b-b435-c99583f74b98 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Terminating instance [ 1231.090406] env[67752]: DEBUG nova.compute.manager [None req-a62a8de1-5e3c-479b-b435-c99583f74b98 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1231.090604] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a62a8de1-5e3c-479b-b435-c99583f74b98 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1231.091109] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-19a2c3d0-1e4c-48c0-9d88-5344cc24e405 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1231.095604] env[67752]: DEBUG nova.compute.manager [None req-db8f6f02-176a-4689-a251-166bd122b16f tempest-ListServersNegativeTestJSON-684094107 tempest-ListServersNegativeTestJSON-684094107-project-member] [instance: e2fc8dbe-6458-4f5a-9aef-c44ea4d61b08] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1231.102034] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa31a978-f294-4d55-9d3e-8ad4549e2b73 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1231.135669] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-a62a8de1-5e3c-479b-b435-c99583f74b98 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ebd6c719-32ae-410c-a444-db7ad3f8a9c5 could not be found. [ 1231.135977] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a62a8de1-5e3c-479b-b435-c99583f74b98 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1231.136248] env[67752]: INFO nova.compute.manager [None req-a62a8de1-5e3c-479b-b435-c99583f74b98 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1231.136549] env[67752]: DEBUG oslo.service.loopingcall [None req-a62a8de1-5e3c-479b-b435-c99583f74b98 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1231.137673] env[67752]: DEBUG oslo_concurrency.lockutils [None req-db8f6f02-176a-4689-a251-166bd122b16f tempest-ListServersNegativeTestJSON-684094107 tempest-ListServersNegativeTestJSON-684094107-project-member] Lock "9065b97a-05a3-4b62-8f4a-418d5b0b209a" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 213.731s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1231.137906] env[67752]: DEBUG nova.network.neutron [-] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1231.139073] env[67752]: DEBUG nova.compute.manager [None req-db8f6f02-176a-4689-a251-166bd122b16f tempest-ListServersNegativeTestJSON-684094107 tempest-ListServersNegativeTestJSON-684094107-project-member] [instance: e2fc8dbe-6458-4f5a-9aef-c44ea4d61b08] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1231.140236] env[67752]: DEBUG nova.compute.manager [-] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1231.140349] env[67752]: DEBUG nova.network.neutron [-] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1231.150142] env[67752]: INFO nova.compute.manager [-] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] Took 0.07 seconds to deallocate network for instance. [ 1231.166990] env[67752]: DEBUG oslo_concurrency.lockutils [None req-db8f6f02-176a-4689-a251-166bd122b16f tempest-ListServersNegativeTestJSON-684094107 tempest-ListServersNegativeTestJSON-684094107-project-member] Lock "e2fc8dbe-6458-4f5a-9aef-c44ea4d61b08" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 213.688s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1231.171585] env[67752]: DEBUG nova.compute.manager [None req-db8f6f02-176a-4689-a251-166bd122b16f tempest-ListServersNegativeTestJSON-684094107 tempest-ListServersNegativeTestJSON-684094107-project-member] [instance: 6da6a17c-b133-4d3c-8b96-6317760a7b3b] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1231.186788] env[67752]: DEBUG nova.compute.manager [None req-6f4d6066-9124-472e-9aa7-1f9ed12c4f85 tempest-TenantUsagesTestJSON-1814372835 tempest-TenantUsagesTestJSON-1814372835-project-member] [instance: 9f409285-631b-48f9-8b80-a3179c09a237] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1231.199895] env[67752]: DEBUG nova.compute.manager [None req-db8f6f02-176a-4689-a251-166bd122b16f tempest-ListServersNegativeTestJSON-684094107 tempest-ListServersNegativeTestJSON-684094107-project-member] [instance: 6da6a17c-b133-4d3c-8b96-6317760a7b3b] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1231.224121] env[67752]: DEBUG oslo_concurrency.lockutils [None req-db8f6f02-176a-4689-a251-166bd122b16f tempest-ListServersNegativeTestJSON-684094107 tempest-ListServersNegativeTestJSON-684094107-project-member] Lock "6da6a17c-b133-4d3c-8b96-6317760a7b3b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 213.705s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1231.228971] env[67752]: DEBUG nova.compute.manager [None req-6f4d6066-9124-472e-9aa7-1f9ed12c4f85 tempest-TenantUsagesTestJSON-1814372835 tempest-TenantUsagesTestJSON-1814372835-project-member] [instance: 9f409285-631b-48f9-8b80-a3179c09a237] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1231.248602] env[67752]: DEBUG nova.compute.manager [None req-7ce83be0-0ebf-45d2-b9d2-c95afb5fb37f tempest-ServerShowV254Test-2141823222 tempest-ServerShowV254Test-2141823222-project-member] [instance: 3b3fa52f-9512-4de3-8029-5cbafed0b1ae] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1231.266206] env[67752]: DEBUG neutronclient.v2_0.client [-] Error message: {"error": {"code": 401, "title": "Unauthorized", "message": "The request you have made requires authentication."}} {{(pid=67752) _handle_fault_response /opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py:262}} [ 1231.267107] env[67752]: ERROR nova.network.neutron [-] Neutron client was not able to generate a valid admin token, please verify Neutron admin credential located in nova.conf: neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1231.267314] env[67752]: ERROR oslo.service.loopingcall [-] Dynamic interval looping call 'oslo_service.loopingcall.RetryDecorator.__call__.._func' failed: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1231.267314] env[67752]: ERROR oslo.service.loopingcall Traceback (most recent call last): [ 1231.267314] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1231.267314] env[67752]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1231.267314] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1231.267314] env[67752]: ERROR oslo.service.loopingcall exception_handler_v20(status_code, error_body) [ 1231.267314] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1231.267314] env[67752]: ERROR oslo.service.loopingcall raise client_exc(message=error_message, [ 1231.267314] env[67752]: ERROR oslo.service.loopingcall neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1231.267314] env[67752]: ERROR oslo.service.loopingcall Neutron server returns request_ids: ['req-3399da2e-9403-4b10-811d-2749842af3e5'] [ 1231.267314] env[67752]: ERROR oslo.service.loopingcall [ 1231.267314] env[67752]: ERROR oslo.service.loopingcall During handling of the above exception, another exception occurred: [ 1231.267314] env[67752]: ERROR oslo.service.loopingcall [ 1231.267314] env[67752]: ERROR oslo.service.loopingcall Traceback (most recent call last): [ 1231.267314] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1231.267314] env[67752]: ERROR oslo.service.loopingcall result = func(*self.args, **self.kw) [ 1231.267804] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1231.267804] env[67752]: ERROR oslo.service.loopingcall result = f(*args, **kwargs) [ 1231.267804] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/compute/manager.py", line 3062, in _deallocate_network_with_retries [ 1231.267804] env[67752]: ERROR oslo.service.loopingcall self._deallocate_network( [ 1231.267804] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/compute/manager.py", line 2282, in _deallocate_network [ 1231.267804] env[67752]: ERROR oslo.service.loopingcall self.network_api.deallocate_for_instance( [ 1231.267804] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1231.267804] env[67752]: ERROR oslo.service.loopingcall data = neutron.list_ports(**search_opts) [ 1231.267804] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1231.267804] env[67752]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1231.267804] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1231.267804] env[67752]: ERROR oslo.service.loopingcall return self.list('ports', self.ports_path, retrieve_all, [ 1231.267804] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1231.267804] env[67752]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1231.267804] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1231.267804] env[67752]: ERROR oslo.service.loopingcall for r in self._pagination(collection, path, **params): [ 1231.267804] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1231.267804] env[67752]: ERROR oslo.service.loopingcall res = self.get(path, params=params) [ 1231.268785] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1231.268785] env[67752]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1231.268785] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1231.268785] env[67752]: ERROR oslo.service.loopingcall return self.retry_request("GET", action, body=body, [ 1231.268785] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1231.268785] env[67752]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1231.268785] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1231.268785] env[67752]: ERROR oslo.service.loopingcall return self.do_request(method, action, body=body, [ 1231.268785] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1231.268785] env[67752]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 1231.268785] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1231.268785] env[67752]: ERROR oslo.service.loopingcall self._handle_fault_response(status_code, replybody, resp) [ 1231.268785] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1231.268785] env[67752]: ERROR oslo.service.loopingcall raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1231.268785] env[67752]: ERROR oslo.service.loopingcall nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1231.268785] env[67752]: ERROR oslo.service.loopingcall [ 1231.269370] env[67752]: ERROR nova.compute.manager [None req-a62a8de1-5e3c-479b-b435-c99583f74b98 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Failed to deallocate network for instance. Error: Networking client is experiencing an unauthorized exception.: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1231.271388] env[67752]: DEBUG oslo_concurrency.lockutils [None req-6f4d6066-9124-472e-9aa7-1f9ed12c4f85 tempest-TenantUsagesTestJSON-1814372835 tempest-TenantUsagesTestJSON-1814372835-project-member] Lock "9f409285-631b-48f9-8b80-a3179c09a237" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 212.521s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1231.283260] env[67752]: DEBUG nova.compute.manager [None req-7ce83be0-0ebf-45d2-b9d2-c95afb5fb37f tempest-ServerShowV254Test-2141823222 tempest-ServerShowV254Test-2141823222-project-member] [instance: 3b3fa52f-9512-4de3-8029-5cbafed0b1ae] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1231.299181] env[67752]: DEBUG nova.compute.manager [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1231.320018] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4afda383-bd2a-490e-9e57-c8e10d9bd133 tempest-ServersTestJSON-982106617 tempest-ServersTestJSON-982106617-project-member] Lock "02942b29-2343-441e-9e2b-aca2af57c849" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.306s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1231.320018] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "02942b29-2343-441e-9e2b-aca2af57c849" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 51.680s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1231.320018] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 02942b29-2343-441e-9e2b-aca2af57c849] During sync_power_state the instance has a pending task (deleting). Skip. [ 1231.320018] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "02942b29-2343-441e-9e2b-aca2af57c849" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1231.330132] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7ce83be0-0ebf-45d2-b9d2-c95afb5fb37f tempest-ServerShowV254Test-2141823222 tempest-ServerShowV254Test-2141823222-project-member] Lock "3b3fa52f-9512-4de3-8029-5cbafed0b1ae" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 198.201s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1231.335456] env[67752]: ERROR nova.compute.manager [None req-a62a8de1-5e3c-479b-b435-c99583f74b98 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Setting instance vm_state to ERROR: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1231.335456] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Traceback (most recent call last): [ 1231.335456] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1231.335456] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] ret = obj(*args, **kwargs) [ 1231.335456] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1231.335456] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] exception_handler_v20(status_code, error_body) [ 1231.335456] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1231.335456] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] raise client_exc(message=error_message, [ 1231.335456] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1231.335456] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Neutron server returns request_ids: ['req-3399da2e-9403-4b10-811d-2749842af3e5'] [ 1231.335921] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] [ 1231.335921] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] During handling of the above exception, another exception occurred: [ 1231.335921] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] [ 1231.335921] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Traceback (most recent call last): [ 1231.335921] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/compute/manager.py", line 3332, in do_terminate_instance [ 1231.335921] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self._delete_instance(context, instance, bdms) [ 1231.335921] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/compute/manager.py", line 3267, in _delete_instance [ 1231.335921] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self._shutdown_instance(context, instance, bdms) [ 1231.335921] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/compute/manager.py", line 3161, in _shutdown_instance [ 1231.335921] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self._try_deallocate_network(context, instance, requested_networks) [ 1231.335921] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/compute/manager.py", line 3075, in _try_deallocate_network [ 1231.335921] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] with excutils.save_and_reraise_exception(): [ 1231.335921] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1231.335921] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self.force_reraise() [ 1231.336573] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1231.336573] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] raise self.value [ 1231.336573] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/compute/manager.py", line 3073, in _try_deallocate_network [ 1231.336573] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] _deallocate_network_with_retries() [ 1231.336573] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 436, in func [ 1231.336573] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return evt.wait() [ 1231.336573] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1231.336573] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] result = hub.switch() [ 1231.336573] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1231.336573] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self.greenlet.switch() [ 1231.336573] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1231.336573] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] result = func(*self.args, **self.kw) [ 1231.337071] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1231.337071] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] result = f(*args, **kwargs) [ 1231.337071] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/compute/manager.py", line 3062, in _deallocate_network_with_retries [ 1231.337071] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self._deallocate_network( [ 1231.337071] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/compute/manager.py", line 2282, in _deallocate_network [ 1231.337071] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self.network_api.deallocate_for_instance( [ 1231.337071] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1231.337071] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] data = neutron.list_ports(**search_opts) [ 1231.337071] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1231.337071] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] ret = obj(*args, **kwargs) [ 1231.337071] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1231.337071] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self.list('ports', self.ports_path, retrieve_all, [ 1231.337071] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1231.337553] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] ret = obj(*args, **kwargs) [ 1231.337553] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1231.337553] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] for r in self._pagination(collection, path, **params): [ 1231.337553] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1231.337553] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] res = self.get(path, params=params) [ 1231.337553] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1231.337553] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] ret = obj(*args, **kwargs) [ 1231.337553] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1231.337553] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self.retry_request("GET", action, body=body, [ 1231.337553] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1231.337553] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] ret = obj(*args, **kwargs) [ 1231.337553] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1231.337553] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] return self.do_request(method, action, body=body, [ 1231.337984] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1231.337984] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] ret = obj(*args, **kwargs) [ 1231.337984] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1231.337984] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] self._handle_fault_response(status_code, replybody, resp) [ 1231.337984] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1231.337984] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1231.337984] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1231.337984] env[67752]: ERROR nova.compute.manager [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] [ 1231.360260] env[67752]: DEBUG nova.compute.manager [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1231.373055] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a62a8de1-5e3c-479b-b435-c99583f74b98 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Lock "ebd6c719-32ae-410c-a444-db7ad3f8a9c5" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.287s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1231.374370] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "ebd6c719-32ae-410c-a444-db7ad3f8a9c5" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 51.734s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1231.374587] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] During sync_power_state the instance has a pending task (deleting). Skip. [ 1231.375173] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "ebd6c719-32ae-410c-a444-db7ad3f8a9c5" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1231.424490] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1231.424490] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1231.424898] env[67752]: INFO nova.compute.claims [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1231.435912] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1231.454408] env[67752]: INFO nova.compute.manager [None req-a62a8de1-5e3c-479b-b435-c99583f74b98 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] [instance: ebd6c719-32ae-410c-a444-db7ad3f8a9c5] Successfully reverted task state from None on failure for instance. [ 1231.464514] env[67752]: ERROR oslo_messaging.rpc.server [None req-a62a8de1-5e3c-479b-b435-c99583f74b98 tempest-ServersAdminNegativeTestJSON-1667354701 tempest-ServersAdminNegativeTestJSON-1667354701-project-member] Exception during message handling: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1231.464514] env[67752]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): [ 1231.464514] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1231.464514] env[67752]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1231.464514] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 1231.464514] env[67752]: ERROR oslo_messaging.rpc.server exception_handler_v20(status_code, error_body) [ 1231.464514] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 1231.464514] env[67752]: ERROR oslo_messaging.rpc.server raise client_exc(message=error_message, [ 1231.464514] env[67752]: ERROR oslo_messaging.rpc.server neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 1231.464514] env[67752]: ERROR oslo_messaging.rpc.server Neutron server returns request_ids: ['req-3399da2e-9403-4b10-811d-2749842af3e5'] [ 1231.464514] env[67752]: ERROR oslo_messaging.rpc.server [ 1231.464514] env[67752]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: [ 1231.464514] env[67752]: ERROR oslo_messaging.rpc.server [ 1231.464514] env[67752]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): [ 1231.464514] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming [ 1231.464998] env[67752]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) [ 1231.464998] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch [ 1231.464998] env[67752]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) [ 1231.464998] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch [ 1231.464998] env[67752]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) [ 1231.464998] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/exception_wrapper.py", line 65, in wrapped [ 1231.464998] env[67752]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1231.464998] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1231.464998] env[67752]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1231.464998] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1231.464998] env[67752]: ERROR oslo_messaging.rpc.server raise self.value [ 1231.464998] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/exception_wrapper.py", line 63, in wrapped [ 1231.464998] env[67752]: ERROR oslo_messaging.rpc.server return f(self, context, *args, **kw) [ 1231.464998] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 166, in decorated_function [ 1231.464998] env[67752]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1231.464998] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1231.464998] env[67752]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1231.464998] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1231.465529] env[67752]: ERROR oslo_messaging.rpc.server raise self.value [ 1231.465529] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 157, in decorated_function [ 1231.465529] env[67752]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1231.465529] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/utils.py", line 1453, in decorated_function [ 1231.465529] env[67752]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1231.465529] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 213, in decorated_function [ 1231.465529] env[67752]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1231.465529] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1231.465529] env[67752]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1231.465529] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1231.465529] env[67752]: ERROR oslo_messaging.rpc.server raise self.value [ 1231.465529] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 203, in decorated_function [ 1231.465529] env[67752]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 1231.465529] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3344, in terminate_instance [ 1231.465529] env[67752]: ERROR oslo_messaging.rpc.server do_terminate_instance(instance, bdms) [ 1231.465529] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py", line 412, in inner [ 1231.465529] env[67752]: ERROR oslo_messaging.rpc.server return f(*args, **kwargs) [ 1231.465529] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3339, in do_terminate_instance [ 1231.466068] env[67752]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1231.466068] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1231.466068] env[67752]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1231.466068] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1231.466068] env[67752]: ERROR oslo_messaging.rpc.server raise self.value [ 1231.466068] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3332, in do_terminate_instance [ 1231.466068] env[67752]: ERROR oslo_messaging.rpc.server self._delete_instance(context, instance, bdms) [ 1231.466068] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3267, in _delete_instance [ 1231.466068] env[67752]: ERROR oslo_messaging.rpc.server self._shutdown_instance(context, instance, bdms) [ 1231.466068] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3161, in _shutdown_instance [ 1231.466068] env[67752]: ERROR oslo_messaging.rpc.server self._try_deallocate_network(context, instance, requested_networks) [ 1231.466068] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3075, in _try_deallocate_network [ 1231.466068] env[67752]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 1231.466068] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 1231.466068] env[67752]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 1231.466068] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 1231.466068] env[67752]: ERROR oslo_messaging.rpc.server raise self.value [ 1231.466068] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3073, in _try_deallocate_network [ 1231.466617] env[67752]: ERROR oslo_messaging.rpc.server _deallocate_network_with_retries() [ 1231.466617] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 436, in func [ 1231.466617] env[67752]: ERROR oslo_messaging.rpc.server return evt.wait() [ 1231.466617] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1231.466617] env[67752]: ERROR oslo_messaging.rpc.server result = hub.switch() [ 1231.466617] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1231.466617] env[67752]: ERROR oslo_messaging.rpc.server return self.greenlet.switch() [ 1231.466617] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 1231.466617] env[67752]: ERROR oslo_messaging.rpc.server result = func(*self.args, **self.kw) [ 1231.466617] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 1231.466617] env[67752]: ERROR oslo_messaging.rpc.server result = f(*args, **kwargs) [ 1231.466617] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3062, in _deallocate_network_with_retries [ 1231.466617] env[67752]: ERROR oslo_messaging.rpc.server self._deallocate_network( [ 1231.466617] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 2282, in _deallocate_network [ 1231.466617] env[67752]: ERROR oslo_messaging.rpc.server self.network_api.deallocate_for_instance( [ 1231.466617] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 1231.466617] env[67752]: ERROR oslo_messaging.rpc.server data = neutron.list_ports(**search_opts) [ 1231.466617] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1231.467149] env[67752]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1231.467149] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 1231.467149] env[67752]: ERROR oslo_messaging.rpc.server return self.list('ports', self.ports_path, retrieve_all, [ 1231.467149] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1231.467149] env[67752]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1231.467149] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 1231.467149] env[67752]: ERROR oslo_messaging.rpc.server for r in self._pagination(collection, path, **params): [ 1231.467149] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 1231.467149] env[67752]: ERROR oslo_messaging.rpc.server res = self.get(path, params=params) [ 1231.467149] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1231.467149] env[67752]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1231.467149] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 1231.467149] env[67752]: ERROR oslo_messaging.rpc.server return self.retry_request("GET", action, body=body, [ 1231.467149] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1231.467149] env[67752]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1231.467149] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 1231.467149] env[67752]: ERROR oslo_messaging.rpc.server return self.do_request(method, action, body=body, [ 1231.467149] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 1231.467652] env[67752]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 1231.467652] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 1231.467652] env[67752]: ERROR oslo_messaging.rpc.server self._handle_fault_response(status_code, replybody, resp) [ 1231.467652] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 1231.467652] env[67752]: ERROR oslo_messaging.rpc.server raise exception.NeutronAdminCredentialConfigurationInvalid() [ 1231.467652] env[67752]: ERROR oslo_messaging.rpc.server nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 1231.467652] env[67752]: ERROR oslo_messaging.rpc.server [ 1231.787947] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-140a57a5-dbe5-4daf-9742-69edc0a66585 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1231.796108] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-23abfab6-f5f6-46e6-bf20-bda19be5875d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1231.828313] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-72b9782a-7161-4029-b46c-080c7567dc10 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1231.836506] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c43518f-4616-499d-8abe-1378b656de64 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1231.849729] env[67752]: DEBUG nova.compute.provider_tree [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1231.858161] env[67752]: DEBUG nova.scheduler.client.report [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1231.871977] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.450s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1231.872846] env[67752]: DEBUG nova.compute.manager [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1231.875077] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.439s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1231.876922] env[67752]: INFO nova.compute.claims [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1231.917967] env[67752]: DEBUG nova.compute.utils [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1231.922910] env[67752]: DEBUG nova.compute.manager [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1231.922910] env[67752]: DEBUG nova.network.neutron [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1231.927715] env[67752]: DEBUG nova.compute.manager [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1231.970291] env[67752]: INFO nova.virt.block_device [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Booting with volume 9d573ff9-782c-4764-a4d9-b726cc147a0c at /dev/sda [ 1232.026114] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-37e85fb9-2585-44a7-a449-fed1bb9f04ee {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1232.034218] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8bc5a456-2edb-461f-a52e-ce5e31bf4565 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1232.071979] env[67752]: DEBUG nova.policy [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'b98986cfb1924614ab4b4465cbd330db', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '09c4ffbb081d4e099cd33ffed072a63b', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1232.074624] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-e479651d-d030-4292-a57e-2064fbd21f1d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1232.086658] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cce449cb-d860-493d-8a6c-3e4389a68dbe {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1232.123813] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab0af1b0-fe89-4b42-9473-31c2943536c2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1232.131756] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0e4608e-d487-4200-bf76-35218748d19a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1232.144674] env[67752]: DEBUG nova.virt.block_device [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Updating existing volume attachment record: 094bb065-8827-4615-bd4b-b3105e938364 {{(pid=67752) _volume_attach /opt/stack/nova/nova/virt/block_device.py:665}} [ 1232.313248] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b6c2fee2-ba44-41a0-b71e-1c19b92b7f8f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1232.320708] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba754507-564b-433d-9628-b7f392160d86 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1232.355958] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-baf88319-2248-4596-9b9c-717654a06121 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1232.363836] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7871ec83-f315-44a1-8572-7e8fafccc687 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1232.378632] env[67752]: DEBUG nova.compute.provider_tree [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1232.392015] env[67752]: DEBUG nova.scheduler.client.report [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1232.421090] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.544s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1232.421090] env[67752]: DEBUG nova.compute.manager [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1232.449688] env[67752]: DEBUG nova.compute.manager [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1232.449998] env[67752]: DEBUG nova.virt.hardware [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum=,container_format=,created_at=,direct_url=,disk_format=,id=,min_disk=0,min_ram=0,name=,owner=,properties=ImageMetaProps,protected=,size=1073741824,status='active',tags=,updated_at=,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1232.450487] env[67752]: DEBUG nova.virt.hardware [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1232.450487] env[67752]: DEBUG nova.virt.hardware [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1232.450629] env[67752]: DEBUG nova.virt.hardware [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1232.450694] env[67752]: DEBUG nova.virt.hardware [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1232.450849] env[67752]: DEBUG nova.virt.hardware [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1232.451091] env[67752]: DEBUG nova.virt.hardware [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1232.451858] env[67752]: DEBUG nova.virt.hardware [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1232.451858] env[67752]: DEBUG nova.virt.hardware [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1232.451858] env[67752]: DEBUG nova.virt.hardware [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1232.451858] env[67752]: DEBUG nova.virt.hardware [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1232.452953] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ce90d44-35ba-417d-9228-4ef816408051 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1232.461714] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cfba73fc-8c55-4e20-a352-abf041b47b00 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1232.467090] env[67752]: DEBUG nova.compute.utils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1232.468920] env[67752]: DEBUG nova.compute.manager [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1232.469161] env[67752]: DEBUG nova.network.neutron [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1232.479903] env[67752]: DEBUG nova.compute.manager [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1232.563922] env[67752]: DEBUG nova.compute.manager [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1232.596190] env[67752]: DEBUG nova.virt.hardware [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1232.596190] env[67752]: DEBUG nova.virt.hardware [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1232.596190] env[67752]: DEBUG nova.virt.hardware [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1232.596372] env[67752]: DEBUG nova.virt.hardware [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1232.596372] env[67752]: DEBUG nova.virt.hardware [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1232.596892] env[67752]: DEBUG nova.virt.hardware [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1232.597399] env[67752]: DEBUG nova.virt.hardware [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1232.600068] env[67752]: DEBUG nova.virt.hardware [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1232.600068] env[67752]: DEBUG nova.virt.hardware [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1232.600068] env[67752]: DEBUG nova.virt.hardware [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1232.600068] env[67752]: DEBUG nova.virt.hardware [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1232.600068] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-30c449d4-7900-4b6b-8c74-b189928b3bbd {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1232.609918] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c76c4f36-be41-4c0c-9818-737670a9ea07 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1232.627995] env[67752]: DEBUG nova.policy [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '641d021b5cfb4a3885a3db7932407a38', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e117ebfe10cb4714b149d6e9bc132273', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1233.297171] env[67752]: DEBUG nova.network.neutron [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Successfully created port: d8fdd994-dcd0-410c-90fb-dc609c98d82b {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1234.147986] env[67752]: DEBUG nova.network.neutron [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Successfully created port: 3fae749a-47d0-4a0d-880f-fa2cbe6eaed7 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1235.587384] env[67752]: DEBUG nova.network.neutron [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Successfully updated port: d8fdd994-dcd0-410c-90fb-dc609c98d82b {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1235.591623] env[67752]: DEBUG nova.compute.manager [req-b69f4275-06f1-48f9-8c95-b7237cb76712 req-20487960-cd24-4769-a6c6-f21b24a225a1 service nova] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Received event network-vif-plugged-d8fdd994-dcd0-410c-90fb-dc609c98d82b {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1235.591877] env[67752]: DEBUG oslo_concurrency.lockutils [req-b69f4275-06f1-48f9-8c95-b7237cb76712 req-20487960-cd24-4769-a6c6-f21b24a225a1 service nova] Acquiring lock "7c8fc8d3-1f51-4793-8954-93451ebffa74-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1235.592647] env[67752]: DEBUG oslo_concurrency.lockutils [req-b69f4275-06f1-48f9-8c95-b7237cb76712 req-20487960-cd24-4769-a6c6-f21b24a225a1 service nova] Lock "7c8fc8d3-1f51-4793-8954-93451ebffa74-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1235.592647] env[67752]: DEBUG oslo_concurrency.lockutils [req-b69f4275-06f1-48f9-8c95-b7237cb76712 req-20487960-cd24-4769-a6c6-f21b24a225a1 service nova] Lock "7c8fc8d3-1f51-4793-8954-93451ebffa74-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1235.592647] env[67752]: DEBUG nova.compute.manager [req-b69f4275-06f1-48f9-8c95-b7237cb76712 req-20487960-cd24-4769-a6c6-f21b24a225a1 service nova] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] No waiting events found dispatching network-vif-plugged-d8fdd994-dcd0-410c-90fb-dc609c98d82b {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1235.592647] env[67752]: WARNING nova.compute.manager [req-b69f4275-06f1-48f9-8c95-b7237cb76712 req-20487960-cd24-4769-a6c6-f21b24a225a1 service nova] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Received unexpected event network-vif-plugged-d8fdd994-dcd0-410c-90fb-dc609c98d82b for instance with vm_state building and task_state spawning. [ 1235.600746] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Acquiring lock "refresh_cache-7c8fc8d3-1f51-4793-8954-93451ebffa74" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1235.600885] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Acquired lock "refresh_cache-7c8fc8d3-1f51-4793-8954-93451ebffa74" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1235.601080] env[67752]: DEBUG nova.network.neutron [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1235.689357] env[67752]: DEBUG nova.network.neutron [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1236.411569] env[67752]: DEBUG nova.network.neutron [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Updating instance_info_cache with network_info: [{"id": "d8fdd994-dcd0-410c-90fb-dc609c98d82b", "address": "fa:16:3e:f9:82:4c", "network": {"id": "bedc8a51-9d5d-4005-944e-a476d84af480", "bridge": "br-int", "label": "tempest-ServersTestBootFromVolume-384900801-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "09c4ffbb081d4e099cd33ffed072a63b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dbd2870d-a51d-472a-8034-1b3e132b5cb6", "external-id": "nsx-vlan-transportzone-101", "segmentation_id": 101, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapd8fdd994-dc", "ovs_interfaceid": "d8fdd994-dcd0-410c-90fb-dc609c98d82b", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1236.431647] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Releasing lock "refresh_cache-7c8fc8d3-1f51-4793-8954-93451ebffa74" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1236.432033] env[67752]: DEBUG nova.compute.manager [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Instance network_info: |[{"id": "d8fdd994-dcd0-410c-90fb-dc609c98d82b", "address": "fa:16:3e:f9:82:4c", "network": {"id": "bedc8a51-9d5d-4005-944e-a476d84af480", "bridge": "br-int", "label": "tempest-ServersTestBootFromVolume-384900801-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "09c4ffbb081d4e099cd33ffed072a63b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dbd2870d-a51d-472a-8034-1b3e132b5cb6", "external-id": "nsx-vlan-transportzone-101", "segmentation_id": 101, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapd8fdd994-dc", "ovs_interfaceid": "d8fdd994-dcd0-410c-90fb-dc609c98d82b", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1236.433022] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:f9:82:4c', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'dbd2870d-a51d-472a-8034-1b3e132b5cb6', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'd8fdd994-dcd0-410c-90fb-dc609c98d82b', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1236.446078] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Creating folder: Project (09c4ffbb081d4e099cd33ffed072a63b). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1236.446908] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-2a19b4ae-fed3-42fd-b33e-6f13bffc92c4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1236.461030] env[67752]: WARNING suds.client [-] Web service reported a SOAP processing fault using an unexpected HTTP status code 200. Reporting as an internal server error. [ 1236.461030] env[67752]: DEBUG oslo_vmware.api [-] Fault list: [DuplicateName] {{(pid=67752) _invoke_api /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:337}} [ 1236.461030] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Folder already exists: Project (09c4ffbb081d4e099cd33ffed072a63b). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1599}} [ 1236.462652] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Creating folder: Instances. Parent ref: group-v639784. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1236.463681] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-28f54607-a1ab-42fc-a2fb-4dba8ee11297 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1236.475379] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Created folder: Instances in parent group-v639784. [ 1236.475670] env[67752]: DEBUG oslo.service.loopingcall [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1236.475889] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1236.476112] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-8a82054d-0080-4496-b342-f569bc2d5b35 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1236.500677] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1236.500677] env[67752]: value = "task-3199740" [ 1236.500677] env[67752]: _type = "Task" [ 1236.500677] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1236.511744] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199740, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1236.525557] env[67752]: DEBUG nova.network.neutron [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Successfully updated port: 3fae749a-47d0-4a0d-880f-fa2cbe6eaed7 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1236.545873] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "refresh_cache-3d893e80-bbbb-4e58-bf1f-1edd4bccfd05" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1236.546048] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquired lock "refresh_cache-3d893e80-bbbb-4e58-bf1f-1edd4bccfd05" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1236.546185] env[67752]: DEBUG nova.network.neutron [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1236.619419] env[67752]: DEBUG nova.network.neutron [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1236.916206] env[67752]: DEBUG nova.network.neutron [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Updating instance_info_cache with network_info: [{"id": "3fae749a-47d0-4a0d-880f-fa2cbe6eaed7", "address": "fa:16:3e:cf:4a:20", "network": {"id": "48a676ef-ba7b-476c-8ee5-40eb99a49ec1", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-1953737432-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e117ebfe10cb4714b149d6e9bc132273", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "b36c5ae6-c344-4bd1-8239-29128e2bbfbf", "external-id": "nsx-vlan-transportzone-214", "segmentation_id": 214, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap3fae749a-47", "ovs_interfaceid": "3fae749a-47d0-4a0d-880f-fa2cbe6eaed7", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1236.927361] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Releasing lock "refresh_cache-3d893e80-bbbb-4e58-bf1f-1edd4bccfd05" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1236.927656] env[67752]: DEBUG nova.compute.manager [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Instance network_info: |[{"id": "3fae749a-47d0-4a0d-880f-fa2cbe6eaed7", "address": "fa:16:3e:cf:4a:20", "network": {"id": "48a676ef-ba7b-476c-8ee5-40eb99a49ec1", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-1953737432-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e117ebfe10cb4714b149d6e9bc132273", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "b36c5ae6-c344-4bd1-8239-29128e2bbfbf", "external-id": "nsx-vlan-transportzone-214", "segmentation_id": 214, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap3fae749a-47", "ovs_interfaceid": "3fae749a-47d0-4a0d-880f-fa2cbe6eaed7", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1236.928070] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:cf:4a:20', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'b36c5ae6-c344-4bd1-8239-29128e2bbfbf', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '3fae749a-47d0-4a0d-880f-fa2cbe6eaed7', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1236.935484] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Creating folder: Project (e117ebfe10cb4714b149d6e9bc132273). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1236.936032] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-f4dd470f-e627-4945-91d1-eb35ea30ef80 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1236.946198] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Created folder: Project (e117ebfe10cb4714b149d6e9bc132273) in parent group-v639722. [ 1236.946385] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Creating folder: Instances. Parent ref: group-v639800. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1236.946604] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-3dac3fee-ebac-4431-8f91-c1a190d5f3a2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1236.954940] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Created folder: Instances in parent group-v639800. [ 1236.955326] env[67752]: DEBUG oslo.service.loopingcall [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1236.955492] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1236.955707] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-1b1d0f69-738a-48a4-a61d-669e52e4c87b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1236.979876] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1236.979876] env[67752]: value = "task-3199743" [ 1236.979876] env[67752]: _type = "Task" [ 1236.979876] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1236.986865] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199743, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1237.015253] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199740, 'name': CreateVM_Task, 'duration_secs': 0.282218} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1237.015431] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1237.031301] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Block device information present: {'root_device_name': '/dev/sda', 'image': [], 'ephemerals': [], 'block_device_mapping': [{'connection_info': {'driver_volume_type': 'vmdk', 'data': {'volume': 'vm-639787', 'volume_id': '9d573ff9-782c-4764-a4d9-b726cc147a0c', 'name': 'volume-9d573ff9-782c-4764-a4d9-b726cc147a0c', 'profile_id': None, 'qos_specs': None, 'access_mode': 'rw', 'encrypted': False, 'cacheable': False}, 'status': 'reserved', 'instance': '7c8fc8d3-1f51-4793-8954-93451ebffa74', 'attached_at': '', 'detached_at': '', 'volume_id': '9d573ff9-782c-4764-a4d9-b726cc147a0c', 'serial': '9d573ff9-782c-4764-a4d9-b726cc147a0c'}, 'delete_on_termination': True, 'guest_format': None, 'attachment_id': '094bb065-8827-4615-bd4b-b3105e938364', 'disk_bus': None, 'boot_index': 0, 'device_type': None, 'mount_device': '/dev/sda', 'volume_type': None}], 'swap': None} {{(pid=67752) spawn /opt/stack/nova/nova/virt/vmwareapi/vmops.py:799}} [ 1237.031301] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Root volume attach. Driver type: vmdk {{(pid=67752) attach_root_volume /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:661}} [ 1237.031301] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-48078461-4a26-4ded-ac3a-8d8622a07eb8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1237.039685] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ecbcb85-3605-4557-975e-493ce414bce8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1237.047615] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-49dc4553-bc80-4590-8d5c-4a74d2325749 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1237.056991] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.RelocateVM_Task with opID=oslo.vmware-90b9d5dd-02e7-4c2f-a350-fc2b4403363b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1237.064375] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Waiting for the task: (returnval){ [ 1237.064375] env[67752]: value = "task-3199744" [ 1237.064375] env[67752]: _type = "Task" [ 1237.064375] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1237.074808] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199744, 'name': RelocateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1237.497045] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199743, 'name': CreateVM_Task, 'duration_secs': 0.318681} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1237.497286] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1237.498325] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1237.498325] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1237.499214] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1237.499529] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-6eada8be-208c-4457-b18b-950cfc72129b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1237.509257] env[67752]: DEBUG oslo_vmware.api [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Waiting for the task: (returnval){ [ 1237.509257] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52b262df-6f87-86f9-8c02-647e8b120910" [ 1237.509257] env[67752]: _type = "Task" [ 1237.509257] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1237.518928] env[67752]: DEBUG oslo_vmware.api [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52b262df-6f87-86f9-8c02-647e8b120910, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1237.579847] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199744, 'name': RelocateVM_Task} progress is 40%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1237.719283] env[67752]: DEBUG nova.compute.manager [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Received event network-changed-d8fdd994-dcd0-410c-90fb-dc609c98d82b {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1237.719283] env[67752]: DEBUG nova.compute.manager [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Refreshing instance network info cache due to event network-changed-d8fdd994-dcd0-410c-90fb-dc609c98d82b. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1237.719283] env[67752]: DEBUG oslo_concurrency.lockutils [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] Acquiring lock "refresh_cache-7c8fc8d3-1f51-4793-8954-93451ebffa74" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1237.720997] env[67752]: DEBUG oslo_concurrency.lockutils [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] Acquired lock "refresh_cache-7c8fc8d3-1f51-4793-8954-93451ebffa74" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1237.720997] env[67752]: DEBUG nova.network.neutron [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Refreshing network info cache for port d8fdd994-dcd0-410c-90fb-dc609c98d82b {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1237.917604] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Acquiring lock "7c8fc8d3-1f51-4793-8954-93451ebffa74" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1238.023959] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1238.024485] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1238.024829] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1238.079899] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199744, 'name': RelocateVM_Task} progress is 53%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1238.567388] env[67752]: DEBUG nova.network.neutron [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Updated VIF entry in instance network info cache for port d8fdd994-dcd0-410c-90fb-dc609c98d82b. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1238.570172] env[67752]: DEBUG nova.network.neutron [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Updating instance_info_cache with network_info: [{"id": "d8fdd994-dcd0-410c-90fb-dc609c98d82b", "address": "fa:16:3e:f9:82:4c", "network": {"id": "bedc8a51-9d5d-4005-944e-a476d84af480", "bridge": "br-int", "label": "tempest-ServersTestBootFromVolume-384900801-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "09c4ffbb081d4e099cd33ffed072a63b", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "dbd2870d-a51d-472a-8034-1b3e132b5cb6", "external-id": "nsx-vlan-transportzone-101", "segmentation_id": 101, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapd8fdd994-dc", "ovs_interfaceid": "d8fdd994-dcd0-410c-90fb-dc609c98d82b", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1238.595382] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199744, 'name': RelocateVM_Task} progress is 67%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1238.596529] env[67752]: DEBUG oslo_concurrency.lockutils [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] Releasing lock "refresh_cache-7c8fc8d3-1f51-4793-8954-93451ebffa74" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1238.597255] env[67752]: DEBUG nova.compute.manager [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Received event network-vif-plugged-3fae749a-47d0-4a0d-880f-fa2cbe6eaed7 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1238.597255] env[67752]: DEBUG oslo_concurrency.lockutils [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] Acquiring lock "3d893e80-bbbb-4e58-bf1f-1edd4bccfd05-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1238.597255] env[67752]: DEBUG oslo_concurrency.lockutils [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] Lock "3d893e80-bbbb-4e58-bf1f-1edd4bccfd05-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1238.597433] env[67752]: DEBUG oslo_concurrency.lockutils [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] Lock "3d893e80-bbbb-4e58-bf1f-1edd4bccfd05-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1238.597507] env[67752]: DEBUG nova.compute.manager [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] No waiting events found dispatching network-vif-plugged-3fae749a-47d0-4a0d-880f-fa2cbe6eaed7 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1238.597657] env[67752]: WARNING nova.compute.manager [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Received unexpected event network-vif-plugged-3fae749a-47d0-4a0d-880f-fa2cbe6eaed7 for instance with vm_state building and task_state spawning. [ 1238.598050] env[67752]: DEBUG nova.compute.manager [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Received event network-changed-3fae749a-47d0-4a0d-880f-fa2cbe6eaed7 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1238.598050] env[67752]: DEBUG nova.compute.manager [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Refreshing instance network info cache due to event network-changed-3fae749a-47d0-4a0d-880f-fa2cbe6eaed7. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1238.598155] env[67752]: DEBUG oslo_concurrency.lockutils [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] Acquiring lock "refresh_cache-3d893e80-bbbb-4e58-bf1f-1edd4bccfd05" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1238.598290] env[67752]: DEBUG oslo_concurrency.lockutils [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] Acquired lock "refresh_cache-3d893e80-bbbb-4e58-bf1f-1edd4bccfd05" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1238.601110] env[67752]: DEBUG nova.network.neutron [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Refreshing network info cache for port 3fae749a-47d0-4a0d-880f-fa2cbe6eaed7 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1239.083752] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199744, 'name': RelocateVM_Task} progress is 81%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1239.166545] env[67752]: DEBUG nova.network.neutron [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Updated VIF entry in instance network info cache for port 3fae749a-47d0-4a0d-880f-fa2cbe6eaed7. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1239.166945] env[67752]: DEBUG nova.network.neutron [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Updating instance_info_cache with network_info: [{"id": "3fae749a-47d0-4a0d-880f-fa2cbe6eaed7", "address": "fa:16:3e:cf:4a:20", "network": {"id": "48a676ef-ba7b-476c-8ee5-40eb99a49ec1", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-1953737432-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e117ebfe10cb4714b149d6e9bc132273", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "b36c5ae6-c344-4bd1-8239-29128e2bbfbf", "external-id": "nsx-vlan-transportzone-214", "segmentation_id": 214, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap3fae749a-47", "ovs_interfaceid": "3fae749a-47d0-4a0d-880f-fa2cbe6eaed7", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1239.177880] env[67752]: DEBUG oslo_concurrency.lockutils [req-9d39ef2c-508d-4694-8245-1cc05b346197 req-8b132b91-b508-401e-bfa7-86869c7985ac service nova] Releasing lock "refresh_cache-3d893e80-bbbb-4e58-bf1f-1edd4bccfd05" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1239.580641] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199744, 'name': RelocateVM_Task} progress is 95%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1239.923712] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Acquiring lock "e4ebb0f0-ab58-491f-bc4a-7548bbe415d8" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1239.923977] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Lock "e4ebb0f0-ab58-491f-bc4a-7548bbe415d8" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1240.080530] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199744, 'name': RelocateVM_Task} progress is 98%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1240.580869] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199744, 'name': RelocateVM_Task, 'duration_secs': 3.246415} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1240.581217] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Volume attach. Driver type: vmdk {{(pid=67752) attach_volume /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:439}} [ 1240.581412] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] _attach_volume_vmdk: {'driver_volume_type': 'vmdk', 'data': {'volume': 'vm-639787', 'volume_id': '9d573ff9-782c-4764-a4d9-b726cc147a0c', 'name': 'volume-9d573ff9-782c-4764-a4d9-b726cc147a0c', 'profile_id': None, 'qos_specs': None, 'access_mode': 'rw', 'encrypted': False, 'cacheable': False}, 'status': 'reserved', 'instance': '7c8fc8d3-1f51-4793-8954-93451ebffa74', 'attached_at': '', 'detached_at': '', 'volume_id': '9d573ff9-782c-4764-a4d9-b726cc147a0c', 'serial': '9d573ff9-782c-4764-a4d9-b726cc147a0c'} {{(pid=67752) _attach_volume_vmdk /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:336}} [ 1240.582493] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d0f3f5f-4eeb-40f3-b145-2f8a9148ac4a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1240.597566] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6361012a-9872-4588-b755-3f7c4c9b774a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1240.619088] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Reconfiguring VM instance instance-0000003e to attach disk [datastore2] volume-9d573ff9-782c-4764-a4d9-b726cc147a0c/volume-9d573ff9-782c-4764-a4d9-b726cc147a0c.vmdk or device None with type thin {{(pid=67752) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:81}} [ 1240.619379] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-e16e7e30-e4e6-4727-a481-963db5f374c1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1240.639423] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Waiting for the task: (returnval){ [ 1240.639423] env[67752]: value = "task-3199745" [ 1240.639423] env[67752]: _type = "Task" [ 1240.639423] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1240.646982] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199745, 'name': ReconfigVM_Task} progress is 5%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1241.149172] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199745, 'name': ReconfigVM_Task, 'duration_secs': 0.265296} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1241.149474] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Reconfigured VM instance instance-0000003e to attach disk [datastore2] volume-9d573ff9-782c-4764-a4d9-b726cc147a0c/volume-9d573ff9-782c-4764-a4d9-b726cc147a0c.vmdk or device None with type thin {{(pid=67752) attach_disk_to_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:88}} [ 1241.154150] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-2c2ea04b-cfe3-4aa4-a709-1c36d6afcaa2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1241.169172] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Waiting for the task: (returnval){ [ 1241.169172] env[67752]: value = "task-3199746" [ 1241.169172] env[67752]: _type = "Task" [ 1241.169172] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1241.177719] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199746, 'name': ReconfigVM_Task} progress is 5%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1241.678826] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199746, 'name': ReconfigVM_Task, 'duration_secs': 0.122201} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1241.679222] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Attached VMDK: {'driver_volume_type': 'vmdk', 'data': {'volume': 'vm-639787', 'volume_id': '9d573ff9-782c-4764-a4d9-b726cc147a0c', 'name': 'volume-9d573ff9-782c-4764-a4d9-b726cc147a0c', 'profile_id': None, 'qos_specs': None, 'access_mode': 'rw', 'encrypted': False, 'cacheable': False}, 'status': 'reserved', 'instance': '7c8fc8d3-1f51-4793-8954-93451ebffa74', 'attached_at': '', 'detached_at': '', 'volume_id': '9d573ff9-782c-4764-a4d9-b726cc147a0c', 'serial': '9d573ff9-782c-4764-a4d9-b726cc147a0c'} {{(pid=67752) _attach_volume_vmdk /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:361}} [ 1241.679790] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.Rename_Task with opID=oslo.vmware-31eb26c5-47e0-4277-9729-bfe06de9fbc2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1241.685632] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Waiting for the task: (returnval){ [ 1241.685632] env[67752]: value = "task-3199747" [ 1241.685632] env[67752]: _type = "Task" [ 1241.685632] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1241.692839] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199747, 'name': Rename_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1242.197771] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199747, 'name': Rename_Task, 'duration_secs': 0.146677} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1242.198058] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Powering on the VM {{(pid=67752) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1442}} [ 1242.198305] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOnVM_Task with opID=oslo.vmware-205e7e38-cc4d-48ff-941b-4f14213a3252 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1242.204181] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Waiting for the task: (returnval){ [ 1242.204181] env[67752]: value = "task-3199748" [ 1242.204181] env[67752]: _type = "Task" [ 1242.204181] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1242.211495] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199748, 'name': PowerOnVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1242.714803] env[67752]: DEBUG oslo_vmware.api [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199748, 'name': PowerOnVM_Task, 'duration_secs': 0.468788} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1242.715106] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Powered on the VM {{(pid=67752) power_on_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1448}} [ 1242.715295] env[67752]: INFO nova.compute.manager [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Took 10.27 seconds to spawn the instance on the hypervisor. [ 1242.715470] env[67752]: DEBUG nova.compute.manager [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Checking state {{(pid=67752) _get_power_state /opt/stack/nova/nova/compute/manager.py:1782}} [ 1242.716324] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-296d2434-312f-49ca-8fb5-699e24ae8e84 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1242.759910] env[67752]: DEBUG nova.compute.utils [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Conflict updating instance 7c8fc8d3-1f51-4793-8954-93451ebffa74. Expected: {'task_state': ['spawning']}. Actual: {'task_state': 'deleting'} {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1242.761619] env[67752]: DEBUG nova.compute.manager [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Instance disappeared during build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2504}} [ 1242.761786] env[67752]: DEBUG nova.compute.manager [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1242.761953] env[67752]: DEBUG nova.compute.manager [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1242.762142] env[67752]: DEBUG nova.compute.manager [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1242.762306] env[67752]: DEBUG nova.network.neutron [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1243.058274] env[67752]: DEBUG nova.network.neutron [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1243.070387] env[67752]: INFO nova.compute.manager [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Took 0.31 seconds to deallocate network for instance. [ 1243.134204] env[67752]: DEBUG nova.compute.manager [req-6e984756-e6c7-4602-86fe-1293a1fa884d req-468a52cc-e8f9-4978-81d1-63c71b1bb486 service nova] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Received event network-vif-deleted-d8fdd994-dcd0-410c-90fb-dc609c98d82b {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1243.164670] env[67752]: INFO nova.scheduler.client.report [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Deleted allocations for instance 7c8fc8d3-1f51-4793-8954-93451ebffa74 [ 1243.164970] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f320e5e7-11d4-4483-b4ee-8bb681608765 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Lock "7c8fc8d3-1f51-4793-8954-93451ebffa74" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 201.581s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1243.166540] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Lock "7c8fc8d3-1f51-4793-8954-93451ebffa74" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 5.249s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1243.166907] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Acquiring lock "7c8fc8d3-1f51-4793-8954-93451ebffa74-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1243.167158] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Lock "7c8fc8d3-1f51-4793-8954-93451ebffa74-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1243.167364] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Lock "7c8fc8d3-1f51-4793-8954-93451ebffa74-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1243.169781] env[67752]: INFO nova.compute.manager [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Terminating instance [ 1243.171750] env[67752]: DEBUG nova.compute.manager [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1243.171969] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Powering off the VM {{(pid=67752) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1502}} [ 1243.172437] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.PowerOffVM_Task with opID=oslo.vmware-30d30a3a-f0ad-4bba-a0c8-18d07fe4f5b9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.175195] env[67752]: DEBUG nova.compute.manager [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1243.183794] env[67752]: DEBUG oslo_vmware.api [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Waiting for the task: (returnval){ [ 1243.183794] env[67752]: value = "task-3199749" [ 1243.183794] env[67752]: _type = "Task" [ 1243.183794] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1243.192858] env[67752]: DEBUG oslo_vmware.api [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199749, 'name': PowerOffVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1243.225554] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1243.225818] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1243.227285] env[67752]: INFO nova.compute.claims [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1243.518013] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8bfa49d-bc08-4937-a07e-df19e8c6097d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.525593] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ce4061a-9806-44f3-a91a-b26a06e8a016 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.556740] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e2e8c7eb-f851-4218-9cce-aac21d43b120 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.563659] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-331bd947-7310-418f-a760-6dd3b31536ac {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.576248] env[67752]: DEBUG nova.compute.provider_tree [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1243.601306] env[67752]: ERROR nova.scheduler.client.report [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [req-30a501ab-88e9-4589-b90c-63d5cfd34c40] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID c7141b98-bb6c-46bd-b37f-a8e913940199. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-30a501ab-88e9-4589-b90c-63d5cfd34c40"}]} [ 1243.616887] env[67752]: DEBUG nova.scheduler.client.report [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Refreshing inventories for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:804}} [ 1243.630834] env[67752]: DEBUG nova.scheduler.client.report [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Updating ProviderTree inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:768}} [ 1243.631078] env[67752]: DEBUG nova.compute.provider_tree [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1243.643318] env[67752]: DEBUG nova.scheduler.client.report [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Refreshing aggregate associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, aggregates: None {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:813}} [ 1243.659734] env[67752]: DEBUG nova.scheduler.client.report [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Refreshing trait associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, traits: COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:825}} [ 1243.695540] env[67752]: DEBUG oslo_vmware.api [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199749, 'name': PowerOffVM_Task, 'duration_secs': 0.190155} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1243.696306] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Powered off the VM {{(pid=67752) power_off_instance /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1507}} [ 1243.696306] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Volume detach. Driver type: vmdk {{(pid=67752) detach_volume /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:646}} [ 1243.696466] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] _detach_volume_vmdk: {'driver_volume_type': 'vmdk', 'data': {'volume': 'vm-639787', 'volume_id': '9d573ff9-782c-4764-a4d9-b726cc147a0c', 'name': 'volume-9d573ff9-782c-4764-a4d9-b726cc147a0c', 'profile_id': None, 'qos_specs': None, 'access_mode': 'rw', 'encrypted': False, 'cacheable': False}, 'status': 'reserved', 'instance': '7c8fc8d3-1f51-4793-8954-93451ebffa74', 'attached_at': '', 'detached_at': '', 'volume_id': '9d573ff9-782c-4764-a4d9-b726cc147a0c', 'serial': '9d573ff9-782c-4764-a4d9-b726cc147a0c'} {{(pid=67752) _detach_volume_vmdk /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:571}} [ 1243.697246] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7af52fa0-26ce-4a26-be2e-d958ecb78362 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.720523] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec363669-e507-411e-938c-c5d1b5f0a94f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.727138] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f43655e2-1d1f-4479-acce-68e479a8ea17 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.748735] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eab69668-8f16-4124-9b9a-132741f767d8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.765688] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] The volume has not been displaced from its original location: [datastore2] volume-9d573ff9-782c-4764-a4d9-b726cc147a0c/volume-9d573ff9-782c-4764-a4d9-b726cc147a0c.vmdk. No consolidation needed. {{(pid=67752) _consolidate_vmdk_volume /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:504}} [ 1243.770896] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Reconfiguring VM instance instance-0000003e to detach disk 2000 {{(pid=67752) detach_disk_from_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:122}} [ 1243.773430] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-42b99a7f-2759-45f6-83a9-d52d6af1b9ca {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.792995] env[67752]: DEBUG oslo_vmware.api [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Waiting for the task: (returnval){ [ 1243.792995] env[67752]: value = "task-3199750" [ 1243.792995] env[67752]: _type = "Task" [ 1243.792995] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1243.803547] env[67752]: DEBUG oslo_vmware.api [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199750, 'name': ReconfigVM_Task} progress is 10%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1243.913157] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f615aab7-2485-44a2-9977-f6502eacc97f tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "3d893e80-bbbb-4e58-bf1f-1edd4bccfd05" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1243.950998] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-458fd88b-2c1f-4d3c-9fc2-0c6f613fb160 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.958829] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c0acaf41-ff25-45ff-b9d8-abab4726db03 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.989320] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56daea5b-b7dd-46e9-802d-e9b05591bb66 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1243.996376] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-516421c3-8bf6-4102-9fad-0d60de650552 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1244.009170] env[67752]: DEBUG nova.compute.provider_tree [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1244.044685] env[67752]: DEBUG nova.scheduler.client.report [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Updated inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with generation 87 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:957}} [ 1244.044971] env[67752]: DEBUG nova.compute.provider_tree [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Updating resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 generation from 87 to 88 during operation: update_inventory {{(pid=67752) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 1244.045182] env[67752]: DEBUG nova.compute.provider_tree [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1244.059046] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.833s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1244.059527] env[67752]: DEBUG nova.compute.manager [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1244.090134] env[67752]: DEBUG nova.compute.utils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1244.091722] env[67752]: DEBUG nova.compute.manager [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1244.091722] env[67752]: DEBUG nova.network.neutron [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1244.100616] env[67752]: DEBUG nova.compute.manager [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1244.160030] env[67752]: DEBUG nova.policy [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '830b314aad674aabb26153bd49a3da33', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '0d0a7465b94546d59537e84b3ce6d532', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1244.166510] env[67752]: DEBUG nova.compute.manager [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1244.191789] env[67752]: DEBUG nova.virt.hardware [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1244.192088] env[67752]: DEBUG nova.virt.hardware [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1244.192254] env[67752]: DEBUG nova.virt.hardware [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1244.192436] env[67752]: DEBUG nova.virt.hardware [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1244.192587] env[67752]: DEBUG nova.virt.hardware [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1244.192739] env[67752]: DEBUG nova.virt.hardware [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1244.192947] env[67752]: DEBUG nova.virt.hardware [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1244.193113] env[67752]: DEBUG nova.virt.hardware [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1244.193285] env[67752]: DEBUG nova.virt.hardware [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1244.193449] env[67752]: DEBUG nova.virt.hardware [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1244.193622] env[67752]: DEBUG nova.virt.hardware [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1244.194530] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-39bd5064-1c26-4709-a382-078ecda1a9aa {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1244.202922] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d8bc21f8-f28c-4278-a095-c4a7c5484a9c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1244.303627] env[67752]: DEBUG oslo_vmware.api [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199750, 'name': ReconfigVM_Task, 'duration_secs': 0.145624} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1244.303906] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Reconfigured VM instance instance-0000003e to detach disk 2000 {{(pid=67752) detach_disk_from_vm /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:127}} [ 1244.308490] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.ReconfigVM_Task with opID=oslo.vmware-9b5fb9e6-ec01-434e-b707-7204e81eab08 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1244.322825] env[67752]: DEBUG oslo_vmware.api [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Waiting for the task: (returnval){ [ 1244.322825] env[67752]: value = "task-3199751" [ 1244.322825] env[67752]: _type = "Task" [ 1244.322825] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1244.330613] env[67752]: DEBUG oslo_vmware.api [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199751, 'name': ReconfigVM_Task} progress is 5%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1244.832968] env[67752]: DEBUG oslo_vmware.api [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199751, 'name': ReconfigVM_Task, 'duration_secs': 0.095103} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1244.833378] env[67752]: DEBUG nova.virt.vmwareapi.volumeops [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Detached VMDK: {'driver_volume_type': 'vmdk', 'data': {'volume': 'vm-639787', 'volume_id': '9d573ff9-782c-4764-a4d9-b726cc147a0c', 'name': 'volume-9d573ff9-782c-4764-a4d9-b726cc147a0c', 'profile_id': None, 'qos_specs': None, 'access_mode': 'rw', 'encrypted': False, 'cacheable': False}, 'status': 'reserved', 'instance': '7c8fc8d3-1f51-4793-8954-93451ebffa74', 'attached_at': '', 'detached_at': '', 'volume_id': '9d573ff9-782c-4764-a4d9-b726cc147a0c', 'serial': '9d573ff9-782c-4764-a4d9-b726cc147a0c'} {{(pid=67752) _detach_volume_vmdk /opt/stack/nova/nova/virt/vmwareapi/volumeops.py:605}} [ 1244.833616] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1244.834399] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-84c2edee-44fd-4983-903a-9845415426a2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1244.841018] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1244.841258] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-49983f7d-0103-423e-ac66-a21804864bdf {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1244.891820] env[67752]: DEBUG nova.network.neutron [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Successfully created port: 08462e2d-d081-4dc5-b94b-59d1d332f4c8 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1244.895220] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1244.895419] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1244.895598] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Deleting the datastore file [datastore2] 7c8fc8d3-1f51-4793-8954-93451ebffa74 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1244.895845] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-726afe95-639c-4719-81ba-a3e8e88df867 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1244.901864] env[67752]: DEBUG oslo_vmware.api [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Waiting for the task: (returnval){ [ 1244.901864] env[67752]: value = "task-3199753" [ 1244.901864] env[67752]: _type = "Task" [ 1244.901864] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1244.909291] env[67752]: DEBUG oslo_vmware.api [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199753, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1245.411779] env[67752]: DEBUG oslo_vmware.api [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Task: {'id': task-3199753, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.078128} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1245.412052] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1245.412241] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1245.412421] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1245.412601] env[67752]: INFO nova.compute.manager [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Took 2.24 seconds to destroy the instance on the hypervisor. [ 1245.412847] env[67752]: DEBUG oslo.service.loopingcall [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1245.413041] env[67752]: DEBUG nova.compute.manager [-] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1245.413353] env[67752]: DEBUG nova.network.neutron [-] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1245.448274] env[67752]: DEBUG nova.network.neutron [-] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1245.459090] env[67752]: INFO nova.compute.manager [-] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Took 0.05 seconds to deallocate network for instance. [ 1245.527421] env[67752]: INFO nova.compute.manager [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Took 0.07 seconds to detach 1 volumes for instance. [ 1245.530075] env[67752]: DEBUG nova.compute.manager [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Deleting volume: 9d573ff9-782c-4764-a4d9-b726cc147a0c {{(pid=67752) _cleanup_volumes /opt/stack/nova/nova/compute/manager.py:3238}} [ 1245.616349] env[67752]: DEBUG nova.compute.manager [req-9ca258c5-b756-46f6-aef1-46db66482f02 req-ea2712e5-f81f-47cc-9550-2326bd7cd7ee service nova] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Received event network-vif-plugged-08462e2d-d081-4dc5-b94b-59d1d332f4c8 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1245.616349] env[67752]: DEBUG oslo_concurrency.lockutils [req-9ca258c5-b756-46f6-aef1-46db66482f02 req-ea2712e5-f81f-47cc-9550-2326bd7cd7ee service nova] Acquiring lock "a8bd310e-530a-46ab-add7-1b827ea5f399-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1245.616349] env[67752]: DEBUG oslo_concurrency.lockutils [req-9ca258c5-b756-46f6-aef1-46db66482f02 req-ea2712e5-f81f-47cc-9550-2326bd7cd7ee service nova] Lock "a8bd310e-530a-46ab-add7-1b827ea5f399-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1245.616349] env[67752]: DEBUG oslo_concurrency.lockutils [req-9ca258c5-b756-46f6-aef1-46db66482f02 req-ea2712e5-f81f-47cc-9550-2326bd7cd7ee service nova] Lock "a8bd310e-530a-46ab-add7-1b827ea5f399-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1245.617361] env[67752]: DEBUG nova.compute.manager [req-9ca258c5-b756-46f6-aef1-46db66482f02 req-ea2712e5-f81f-47cc-9550-2326bd7cd7ee service nova] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] No waiting events found dispatching network-vif-plugged-08462e2d-d081-4dc5-b94b-59d1d332f4c8 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1245.617361] env[67752]: WARNING nova.compute.manager [req-9ca258c5-b756-46f6-aef1-46db66482f02 req-ea2712e5-f81f-47cc-9550-2326bd7cd7ee service nova] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Received unexpected event network-vif-plugged-08462e2d-d081-4dc5-b94b-59d1d332f4c8 for instance with vm_state building and task_state spawning. [ 1245.617726] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.update_usage" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1245.617943] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1245.618156] env[67752]: DEBUG nova.objects.instance [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Lazy-loading 'resources' on Instance uuid 7c8fc8d3-1f51-4793-8954-93451ebffa74 {{(pid=67752) obj_load_attr /opt/stack/nova/nova/objects/instance.py:1141}} [ 1245.780377] env[67752]: DEBUG nova.network.neutron [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Successfully updated port: 08462e2d-d081-4dc5-b94b-59d1d332f4c8 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1245.789768] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Acquiring lock "refresh_cache-a8bd310e-530a-46ab-add7-1b827ea5f399" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1245.789918] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Acquired lock "refresh_cache-a8bd310e-530a-46ab-add7-1b827ea5f399" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1245.790077] env[67752]: DEBUG nova.network.neutron [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1245.885527] env[67752]: DEBUG nova.network.neutron [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1245.925297] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ae886c1-938c-4ab6-95b4-68d74c68ebc9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1245.934176] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4670748f-d9cc-43f5-832a-7658c94b6f42 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1245.963581] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4d455a0d-1038-4b4d-b053-3e1a7db0e70c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1245.971201] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9a75877b-642a-407f-8b3f-ccb857c00033 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1245.984124] env[67752]: DEBUG nova.compute.provider_tree [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1245.994189] env[67752]: DEBUG nova.scheduler.client.report [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1246.010588] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.update_usage" :: held 0.392s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1246.068732] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea73b363-bbe9-4ba6-b895-ad23db4ad3a2 tempest-ServersTestBootFromVolume-1661551340 tempest-ServersTestBootFromVolume-1661551340-project-member] Lock "7c8fc8d3-1f51-4793-8954-93451ebffa74" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 2.902s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1246.206973] env[67752]: DEBUG nova.network.neutron [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Updating instance_info_cache with network_info: [{"id": "08462e2d-d081-4dc5-b94b-59d1d332f4c8", "address": "fa:16:3e:68:ba:ec", "network": {"id": "1b009bab-fc60-494e-ac52-ea451b1d604f", "bridge": "br-int", "label": "tempest-ImagesTestJSON-1889606404-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "0d0a7465b94546d59537e84b3ce6d532", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "298bb8ef-4765-494c-b157-7a349218bd1e", "external-id": "nsx-vlan-transportzone-905", "segmentation_id": 905, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap08462e2d-d0", "ovs_interfaceid": "08462e2d-d081-4dc5-b94b-59d1d332f4c8", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1246.221503] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Releasing lock "refresh_cache-a8bd310e-530a-46ab-add7-1b827ea5f399" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1246.221812] env[67752]: DEBUG nova.compute.manager [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Instance network_info: |[{"id": "08462e2d-d081-4dc5-b94b-59d1d332f4c8", "address": "fa:16:3e:68:ba:ec", "network": {"id": "1b009bab-fc60-494e-ac52-ea451b1d604f", "bridge": "br-int", "label": "tempest-ImagesTestJSON-1889606404-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "0d0a7465b94546d59537e84b3ce6d532", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "298bb8ef-4765-494c-b157-7a349218bd1e", "external-id": "nsx-vlan-transportzone-905", "segmentation_id": 905, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap08462e2d-d0", "ovs_interfaceid": "08462e2d-d081-4dc5-b94b-59d1d332f4c8", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1246.222232] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:68:ba:ec', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '298bb8ef-4765-494c-b157-7a349218bd1e', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '08462e2d-d081-4dc5-b94b-59d1d332f4c8', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1246.229877] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Creating folder: Project (0d0a7465b94546d59537e84b3ce6d532). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1246.230498] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-2e761d69-93e6-41f6-b986-cb97a42d376b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1246.241309] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Created folder: Project (0d0a7465b94546d59537e84b3ce6d532) in parent group-v639722. [ 1246.241503] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Creating folder: Instances. Parent ref: group-v639803. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1246.241732] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-a21489e7-855c-4f51-8a98-72670d1c8319 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1246.251017] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Created folder: Instances in parent group-v639803. [ 1246.251252] env[67752]: DEBUG oslo.service.loopingcall [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1246.251432] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1246.251625] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-9cbcf7d9-a6b2-4d31-904e-43d95f3acdd3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1246.271579] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1246.271579] env[67752]: value = "task-3199757" [ 1246.271579] env[67752]: _type = "Task" [ 1246.271579] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1246.279325] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199757, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1246.781906] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199757, 'name': CreateVM_Task} progress is 25%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1247.285451] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199757, 'name': CreateVM_Task, 'duration_secs': 0.700961} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1247.285721] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1247.286831] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1247.287158] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1247.288305] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1247.288577] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-2c72fc8b-e869-4c6a-9a0e-e2cb5ac23a85 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1247.298423] env[67752]: DEBUG oslo_vmware.api [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Waiting for the task: (returnval){ [ 1247.298423] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52b667fe-aa83-a69a-993e-95104bce1a9c" [ 1247.298423] env[67752]: _type = "Task" [ 1247.298423] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1247.314469] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1247.314469] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1247.314469] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1247.318688] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0c67db20-a74a-4011-8412-800f0084d087 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Acquiring lock "a8bd310e-530a-46ab-add7-1b827ea5f399" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1247.793094] env[67752]: DEBUG nova.compute.manager [req-d2369a01-317c-4b02-9eb7-263b848bdd7d req-2eca3601-d787-4e6e-9625-5d1dc76d336d service nova] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Received event network-changed-08462e2d-d081-4dc5-b94b-59d1d332f4c8 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1247.793398] env[67752]: DEBUG nova.compute.manager [req-d2369a01-317c-4b02-9eb7-263b848bdd7d req-2eca3601-d787-4e6e-9625-5d1dc76d336d service nova] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Refreshing instance network info cache due to event network-changed-08462e2d-d081-4dc5-b94b-59d1d332f4c8. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1247.793519] env[67752]: DEBUG oslo_concurrency.lockutils [req-d2369a01-317c-4b02-9eb7-263b848bdd7d req-2eca3601-d787-4e6e-9625-5d1dc76d336d service nova] Acquiring lock "refresh_cache-a8bd310e-530a-46ab-add7-1b827ea5f399" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1247.793669] env[67752]: DEBUG oslo_concurrency.lockutils [req-d2369a01-317c-4b02-9eb7-263b848bdd7d req-2eca3601-d787-4e6e-9625-5d1dc76d336d service nova] Acquired lock "refresh_cache-a8bd310e-530a-46ab-add7-1b827ea5f399" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1247.793866] env[67752]: DEBUG nova.network.neutron [req-d2369a01-317c-4b02-9eb7-263b848bdd7d req-2eca3601-d787-4e6e-9625-5d1dc76d336d service nova] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Refreshing network info cache for port 08462e2d-d081-4dc5-b94b-59d1d332f4c8 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1248.158706] env[67752]: DEBUG nova.network.neutron [req-d2369a01-317c-4b02-9eb7-263b848bdd7d req-2eca3601-d787-4e6e-9625-5d1dc76d336d service nova] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Updated VIF entry in instance network info cache for port 08462e2d-d081-4dc5-b94b-59d1d332f4c8. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1248.159097] env[67752]: DEBUG nova.network.neutron [req-d2369a01-317c-4b02-9eb7-263b848bdd7d req-2eca3601-d787-4e6e-9625-5d1dc76d336d service nova] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Updating instance_info_cache with network_info: [{"id": "08462e2d-d081-4dc5-b94b-59d1d332f4c8", "address": "fa:16:3e:68:ba:ec", "network": {"id": "1b009bab-fc60-494e-ac52-ea451b1d604f", "bridge": "br-int", "label": "tempest-ImagesTestJSON-1889606404-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "0d0a7465b94546d59537e84b3ce6d532", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "298bb8ef-4765-494c-b157-7a349218bd1e", "external-id": "nsx-vlan-transportzone-905", "segmentation_id": 905, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap08462e2d-d0", "ovs_interfaceid": "08462e2d-d081-4dc5-b94b-59d1d332f4c8", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1248.169112] env[67752]: DEBUG oslo_concurrency.lockutils [req-d2369a01-317c-4b02-9eb7-263b848bdd7d req-2eca3601-d787-4e6e-9625-5d1dc76d336d service nova] Releasing lock "refresh_cache-a8bd310e-530a-46ab-add7-1b827ea5f399" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1252.408603] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d109a272-e6be-47f6-9b20-86e0f90c6dc3 tempest-ServerRescueNegativeTestJSON-761915415 tempest-ServerRescueNegativeTestJSON-761915415-project-member] Acquiring lock "ba83b143-d752-4166-8ce0-fbd7fd4875a4" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1252.409032] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d109a272-e6be-47f6-9b20-86e0f90c6dc3 tempest-ServerRescueNegativeTestJSON-761915415 tempest-ServerRescueNegativeTestJSON-761915415-project-member] Lock "ba83b143-d752-4166-8ce0-fbd7fd4875a4" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1252.919228] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5f7110-782b-434d-b98d-3b6e08eddc53 tempest-ServerRescueNegativeTestJSON-761915415 tempest-ServerRescueNegativeTestJSON-761915415-project-member] Acquiring lock "6000d3ce-6bdc-4683-a1a9-fbc1e447aad4" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1252.919507] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5f7110-782b-434d-b98d-3b6e08eddc53 tempest-ServerRescueNegativeTestJSON-761915415 tempest-ServerRescueNegativeTestJSON-761915415-project-member] Lock "6000d3ce-6bdc-4683-a1a9-fbc1e447aad4" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1256.021047] env[67752]: DEBUG oslo_concurrency.lockutils [None req-12b68195-bd71-4a5b-9309-cb555652ea5f tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "fe397d83-8625-4c5c-88b2-6c96c51c42bf" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1256.021047] env[67752]: DEBUG oslo_concurrency.lockutils [None req-12b68195-bd71-4a5b-9309-cb555652ea5f tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "fe397d83-8625-4c5c-88b2-6c96c51c42bf" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1258.180482] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae68920d-b3c2-4b1b-a8f3-c069c02ae72d tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] Acquiring lock "9ce8f114-c06d-4f08-b64a-980b94975ff7" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1258.180869] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae68920d-b3c2-4b1b-a8f3-c069c02ae72d tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] Lock "9ce8f114-c06d-4f08-b64a-980b94975ff7" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1259.268776] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1261.948244] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3e9cfa33-5dce-43aa-b42f-3b71ffa58a33 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Acquiring lock "5142518d-3b2a-474c-81d9-d658cb8467e3" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1261.948787] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3e9cfa33-5dce-43aa-b42f-3b71ffa58a33 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Lock "5142518d-3b2a-474c-81d9-d658cb8467e3" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1262.630170] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1262.654363] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1262.654363] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 1262.654543] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 1262.677028] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1262.677028] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1262.677028] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1262.677225] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1262.677225] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1262.677432] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1262.677573] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1262.677690] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: d3940c29-852a-427b-9027-aa4080150724] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1262.677835] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1262.677980] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1262.678162] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 1262.678643] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1262.678868] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1263.635475] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1263.635865] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1263.635965] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1263.636133] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 1266.634928] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1266.647274] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1266.647508] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1266.647698] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1266.647858] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1266.649400] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a4f52dd-d063-4018-b7cc-57dfb15302c5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1266.657931] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4265ffa7-409f-4eea-8fa7-354b75e92cdf {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1266.671686] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a17e8918-e782-42ac-bc33-a3fb2feeba2a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1266.677805] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4652b998-ba98-43ea-87b2-b0c5a3f68666 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1266.708216] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181014MB free_disk=125GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1266.708372] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1266.708572] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1266.782807] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1266.782972] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 411bafb7-ef72-4529-b3f3-0b4955f23788 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1266.783119] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ada70883-9bef-496e-a65f-ca84f6ee6211 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1266.783248] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1266.783370] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b271cd73-a148-48d6-bed9-6a99512457bd actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1266.783492] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1266.783610] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f6499ee5-cb24-44e1-9fe7-c19feeb732eb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1266.783727] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d3940c29-852a-427b-9027-aa4080150724 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1266.783839] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1266.783952] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance a8bd310e-530a-46ab-add7-1b827ea5f399 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1266.795144] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9432cd04-c2df-428b-b31a-93d3ec0c9d1f has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1266.805258] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 90e58033-eacd-433a-8d25-40f7d8403588 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1266.813973] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 81738fe6-6d50-4dfe-ae86-4c8771837168 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1266.822593] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e4ebb0f0-ab58-491f-bc4a-7548bbe415d8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1266.830877] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ba83b143-d752-4166-8ce0-fbd7fd4875a4 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1266.839335] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 6000d3ce-6bdc-4683-a1a9-fbc1e447aad4 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1266.849032] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fe397d83-8625-4c5c-88b2-6c96c51c42bf has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1266.857821] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9ce8f114-c06d-4f08-b64a-980b94975ff7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1266.869452] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 5142518d-3b2a-474c-81d9-d658cb8467e3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1266.869696] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1266.869852] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1267.076977] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd9ad298-ac82-42ff-b10b-792c56e25369 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1267.084497] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bfb7a2e3-099b-42b7-bb2d-7d4ac2ba4f93 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1267.114299] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1c32a81c-c870-4cff-b101-6ba186ad18f4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1267.121173] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67b0289d-ef15-4b98-a78f-34085ead6426 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1267.133809] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1267.141907] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1267.154768] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1267.155037] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.446s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1268.155495] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1279.347780] env[67752]: WARNING oslo_vmware.rw_handles [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1279.347780] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1279.347780] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1279.347780] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1279.347780] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1279.347780] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1279.347780] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1279.347780] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1279.347780] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1279.347780] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1279.347780] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1279.347780] env[67752]: ERROR oslo_vmware.rw_handles [ 1279.348587] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/d529c138-219e-47d1-b4e1-213098dbeead/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1279.350201] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1279.350447] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Copying Virtual Disk [datastore2] vmware_temp/d529c138-219e-47d1-b4e1-213098dbeead/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/d529c138-219e-47d1-b4e1-213098dbeead/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1279.350757] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-7ddb08e2-f69a-4934-b5b3-df3cf2c4bb88 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1279.358962] env[67752]: DEBUG oslo_vmware.api [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Waiting for the task: (returnval){ [ 1279.358962] env[67752]: value = "task-3199758" [ 1279.358962] env[67752]: _type = "Task" [ 1279.358962] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1279.366813] env[67752]: DEBUG oslo_vmware.api [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Task: {'id': task-3199758, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1279.869649] env[67752]: DEBUG oslo_vmware.exceptions [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1279.869994] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1279.870602] env[67752]: ERROR nova.compute.manager [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1279.870602] env[67752]: Faults: ['InvalidArgument'] [ 1279.870602] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Traceback (most recent call last): [ 1279.870602] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1279.870602] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] yield resources [ 1279.870602] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1279.870602] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] self.driver.spawn(context, instance, image_meta, [ 1279.870602] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1279.870602] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1279.870602] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1279.870602] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] self._fetch_image_if_missing(context, vi) [ 1279.870602] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1279.871119] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] image_cache(vi, tmp_image_ds_loc) [ 1279.871119] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1279.871119] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] vm_util.copy_virtual_disk( [ 1279.871119] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1279.871119] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] session._wait_for_task(vmdk_copy_task) [ 1279.871119] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1279.871119] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] return self.wait_for_task(task_ref) [ 1279.871119] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1279.871119] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] return evt.wait() [ 1279.871119] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1279.871119] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] result = hub.switch() [ 1279.871119] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1279.871119] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] return self.greenlet.switch() [ 1279.871501] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1279.871501] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] self.f(*self.args, **self.kw) [ 1279.871501] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1279.871501] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] raise exceptions.translate_fault(task_info.error) [ 1279.871501] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1279.871501] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Faults: ['InvalidArgument'] [ 1279.871501] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] [ 1279.871501] env[67752]: INFO nova.compute.manager [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Terminating instance [ 1279.872552] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1279.872757] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1279.872996] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-e5acbe0d-8055-4a04-8fb3-b3f94425be14 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1279.875421] env[67752]: DEBUG nova.compute.manager [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1279.875613] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1279.876338] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-47600b15-c757-4310-bdfa-3aa0b4304f40 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1279.882774] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1279.883012] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-508b9dbf-d71f-4dc0-b98c-244d642d3a51 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1279.885152] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1279.885325] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1279.886252] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-2d6e58c4-f8c2-4e6a-8a03-300f0738e32f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1279.891140] env[67752]: DEBUG oslo_vmware.api [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Waiting for the task: (returnval){ [ 1279.891140] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52dc9a0e-3bd1-21e1-fe40-b6b80a1e3af0" [ 1279.891140] env[67752]: _type = "Task" [ 1279.891140] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1279.897967] env[67752]: DEBUG oslo_vmware.api [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52dc9a0e-3bd1-21e1-fe40-b6b80a1e3af0, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1279.954716] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1279.955086] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1279.955289] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Deleting the datastore file [datastore2] 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1279.955563] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-0f4574ad-aba7-4718-8266-ba5d4a7c5e10 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1279.962986] env[67752]: DEBUG oslo_vmware.api [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Waiting for the task: (returnval){ [ 1279.962986] env[67752]: value = "task-3199760" [ 1279.962986] env[67752]: _type = "Task" [ 1279.962986] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1279.970746] env[67752]: DEBUG oslo_vmware.api [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Task: {'id': task-3199760, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1280.402036] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1280.402036] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Creating directory with path [datastore2] vmware_temp/1d5bd875-f8d8-4738-b397-7ce1c9799909/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1280.402036] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-9245819e-0e0a-4141-83e6-6b7eaf51381b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1280.415080] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Created directory with path [datastore2] vmware_temp/1d5bd875-f8d8-4738-b397-7ce1c9799909/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1280.415221] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Fetch image to [datastore2] vmware_temp/1d5bd875-f8d8-4738-b397-7ce1c9799909/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1280.415393] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/1d5bd875-f8d8-4738-b397-7ce1c9799909/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1280.416206] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f49d5d40-f6ce-4e52-ac13-37acc65d97fd {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1280.424477] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aee31c60-9c2e-4ca9-b7e3-d6771d2ba078 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1280.433797] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09eb5ea6-307f-4646-a1d7-38a7d5a207b8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1280.467679] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-95cdb797-0723-4d3d-b9a9-fcf10eb44c90 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1280.477624] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-33362db8-a4d3-4bb5-8e01-c081dba1a4a3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1280.479601] env[67752]: DEBUG oslo_vmware.api [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Task: {'id': task-3199760, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.077356} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1280.479807] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1280.479927] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1280.480121] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1280.480299] env[67752]: INFO nova.compute.manager [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1280.482607] env[67752]: DEBUG nova.compute.claims [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1280.482766] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1280.482997] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1280.506069] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1280.562118] env[67752]: DEBUG oslo_vmware.rw_handles [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/1d5bd875-f8d8-4738-b397-7ce1c9799909/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1280.624160] env[67752]: DEBUG oslo_vmware.rw_handles [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1280.624391] env[67752]: DEBUG oslo_vmware.rw_handles [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Closing write handle for https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/1d5bd875-f8d8-4738-b397-7ce1c9799909/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1280.845210] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ba00477-fabd-4efd-bcb8-8705ce3e955b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1280.853890] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac4e03b8-c832-4179-8b0b-3e97d1d97780 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1280.885761] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1dd734c6-fe80-482e-b3ce-e2949dd65329 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1280.893443] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f54720f3-8277-4f61-8723-5511d0e821ea {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1280.906974] env[67752]: DEBUG nova.compute.provider_tree [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1280.937167] env[67752]: ERROR nova.scheduler.client.report [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [req-6eb6629d-5a96-4972-81ed-e72847bf2175] Failed to update inventory to [{'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}}] for resource provider with UUID c7141b98-bb6c-46bd-b37f-a8e913940199. Got 409: {"errors": [{"status": 409, "title": "Conflict", "detail": "There was a conflict when trying to complete your request.\n\n resource provider generation conflict ", "code": "placement.concurrent_update", "request_id": "req-6eb6629d-5a96-4972-81ed-e72847bf2175"}]}: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1280.956807] env[67752]: DEBUG nova.scheduler.client.report [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Refreshing inventories for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:804}} [ 1280.972660] env[67752]: DEBUG nova.scheduler.client.report [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Updating ProviderTree inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:768}} [ 1280.972992] env[67752]: DEBUG nova.compute.provider_tree [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 125, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1280.986471] env[67752]: DEBUG nova.scheduler.client.report [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Refreshing aggregate associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, aggregates: None {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:813}} [ 1281.007301] env[67752]: DEBUG nova.scheduler.client.report [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Refreshing trait associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, traits: COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:825}} [ 1281.257611] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8e5a073c-2f10-4faa-94e6-df144e003b60 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1281.266398] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c7127ea1-5367-44c8-9abf-5eedb839db82 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1281.300031] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e12b4238-6884-428a-9707-4822c904922f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1281.308503] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-35bfbf7a-1714-44ff-841d-a605262496de {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1281.323583] env[67752]: DEBUG nova.compute.provider_tree [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1281.363651] env[67752]: DEBUG nova.scheduler.client.report [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Updated inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with generation 94 in Placement from set_inventory_for_provider using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:957}} [ 1281.363976] env[67752]: DEBUG nova.compute.provider_tree [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Updating resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 generation from 94 to 95 during operation: update_inventory {{(pid=67752) _update_generation /opt/stack/nova/nova/compute/provider_tree.py:164}} [ 1281.364188] env[67752]: DEBUG nova.compute.provider_tree [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1281.379563] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.896s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1281.380188] env[67752]: ERROR nova.compute.manager [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1281.380188] env[67752]: Faults: ['InvalidArgument'] [ 1281.380188] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Traceback (most recent call last): [ 1281.380188] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1281.380188] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] self.driver.spawn(context, instance, image_meta, [ 1281.380188] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1281.380188] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1281.380188] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1281.380188] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] self._fetch_image_if_missing(context, vi) [ 1281.380188] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1281.380188] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] image_cache(vi, tmp_image_ds_loc) [ 1281.380188] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1281.380678] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] vm_util.copy_virtual_disk( [ 1281.380678] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1281.380678] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] session._wait_for_task(vmdk_copy_task) [ 1281.380678] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1281.380678] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] return self.wait_for_task(task_ref) [ 1281.380678] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1281.380678] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] return evt.wait() [ 1281.380678] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1281.380678] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] result = hub.switch() [ 1281.380678] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1281.380678] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] return self.greenlet.switch() [ 1281.380678] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1281.380678] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] self.f(*self.args, **self.kw) [ 1281.381686] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1281.381686] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] raise exceptions.translate_fault(task_info.error) [ 1281.381686] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1281.381686] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Faults: ['InvalidArgument'] [ 1281.381686] env[67752]: ERROR nova.compute.manager [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] [ 1281.381686] env[67752]: DEBUG nova.compute.utils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1281.382601] env[67752]: DEBUG nova.compute.manager [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Build of instance 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9 was re-scheduled: A specified parameter was not correct: fileType [ 1281.382601] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1281.382969] env[67752]: DEBUG nova.compute.manager [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1281.383161] env[67752]: DEBUG nova.compute.manager [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1281.383333] env[67752]: DEBUG nova.compute.manager [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1281.383497] env[67752]: DEBUG nova.network.neutron [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1282.004732] env[67752]: DEBUG nova.network.neutron [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1282.016775] env[67752]: INFO nova.compute.manager [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Took 0.63 seconds to deallocate network for instance. [ 1282.117675] env[67752]: INFO nova.scheduler.client.report [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Deleted allocations for instance 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9 [ 1282.139639] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0383c9c3-2c83-4e8e-ab80-91df53119ca5 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Lock "580c2aa9-160e-40ec-a1e1-0bbaaf60bca9" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 638.982s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1282.140923] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d64f589f-61c3-42d5-84b9-e96830929821 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Lock "580c2aa9-160e-40ec-a1e1-0bbaaf60bca9" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 441.976s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1282.141168] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d64f589f-61c3-42d5-84b9-e96830929821 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Acquiring lock "580c2aa9-160e-40ec-a1e1-0bbaaf60bca9-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1282.141379] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d64f589f-61c3-42d5-84b9-e96830929821 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Lock "580c2aa9-160e-40ec-a1e1-0bbaaf60bca9-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1282.141548] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d64f589f-61c3-42d5-84b9-e96830929821 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Lock "580c2aa9-160e-40ec-a1e1-0bbaaf60bca9-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1282.143887] env[67752]: INFO nova.compute.manager [None req-d64f589f-61c3-42d5-84b9-e96830929821 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Terminating instance [ 1282.145401] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d64f589f-61c3-42d5-84b9-e96830929821 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Acquiring lock "refresh_cache-580c2aa9-160e-40ec-a1e1-0bbaaf60bca9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1282.145539] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d64f589f-61c3-42d5-84b9-e96830929821 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Acquired lock "refresh_cache-580c2aa9-160e-40ec-a1e1-0bbaaf60bca9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1282.145709] env[67752]: DEBUG nova.network.neutron [None req-d64f589f-61c3-42d5-84b9-e96830929821 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1282.160298] env[67752]: DEBUG nova.compute.manager [None req-9889c44c-b170-46af-9567-8a28eff30545 tempest-ServersAdminTestJSON-657333820 tempest-ServersAdminTestJSON-657333820-project-member] [instance: ded2bdfd-71c9-4af4-b595-98ac91fc8203] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1282.185717] env[67752]: DEBUG nova.network.neutron [None req-d64f589f-61c3-42d5-84b9-e96830929821 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1282.194183] env[67752]: DEBUG nova.compute.manager [None req-9889c44c-b170-46af-9567-8a28eff30545 tempest-ServersAdminTestJSON-657333820 tempest-ServersAdminTestJSON-657333820-project-member] [instance: ded2bdfd-71c9-4af4-b595-98ac91fc8203] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1282.214951] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9889c44c-b170-46af-9567-8a28eff30545 tempest-ServersAdminTestJSON-657333820 tempest-ServersAdminTestJSON-657333820-project-member] Lock "ded2bdfd-71c9-4af4-b595-98ac91fc8203" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 227.960s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1282.224205] env[67752]: DEBUG nova.compute.manager [None req-bfcbcd14-66a8-45e4-a9a8-bdc99133a79a tempest-ServersAdminTestJSON-657333820 tempest-ServersAdminTestJSON-657333820-project-member] [instance: 3d423978-c9cf-476f-9df8-d78206dff460] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1282.252263] env[67752]: DEBUG nova.compute.manager [None req-bfcbcd14-66a8-45e4-a9a8-bdc99133a79a tempest-ServersAdminTestJSON-657333820 tempest-ServersAdminTestJSON-657333820-project-member] [instance: 3d423978-c9cf-476f-9df8-d78206dff460] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1282.276544] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bfcbcd14-66a8-45e4-a9a8-bdc99133a79a tempest-ServersAdminTestJSON-657333820 tempest-ServersAdminTestJSON-657333820-project-member] Lock "3d423978-c9cf-476f-9df8-d78206dff460" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 227.325s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1282.290862] env[67752]: DEBUG nova.compute.manager [None req-271904e9-41e9-43d6-b9d8-dd2f1c1e1a11 tempest-ServerShowV257Test-560095070 tempest-ServerShowV257Test-560095070-project-member] [instance: 7c9546af-a9b8-4808-b15e-2ff9d2c12df7] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1282.319342] env[67752]: DEBUG nova.compute.manager [None req-271904e9-41e9-43d6-b9d8-dd2f1c1e1a11 tempest-ServerShowV257Test-560095070 tempest-ServerShowV257Test-560095070-project-member] [instance: 7c9546af-a9b8-4808-b15e-2ff9d2c12df7] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1282.346713] env[67752]: DEBUG oslo_concurrency.lockutils [None req-271904e9-41e9-43d6-b9d8-dd2f1c1e1a11 tempest-ServerShowV257Test-560095070 tempest-ServerShowV257Test-560095070-project-member] Lock "7c9546af-a9b8-4808-b15e-2ff9d2c12df7" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 225.418s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1282.357096] env[67752]: DEBUG nova.compute.manager [None req-50f8be6d-a6fa-472c-a580-8749396bef71 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 78e8ccdd-e5ed-4b87-af85-498ca755561b] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1282.377029] env[67752]: DEBUG nova.network.neutron [None req-d64f589f-61c3-42d5-84b9-e96830929821 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1282.382480] env[67752]: DEBUG nova.compute.manager [None req-50f8be6d-a6fa-472c-a580-8749396bef71 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 78e8ccdd-e5ed-4b87-af85-498ca755561b] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1282.386758] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d64f589f-61c3-42d5-84b9-e96830929821 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Releasing lock "refresh_cache-580c2aa9-160e-40ec-a1e1-0bbaaf60bca9" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1282.387159] env[67752]: DEBUG nova.compute.manager [None req-d64f589f-61c3-42d5-84b9-e96830929821 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1282.387357] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d64f589f-61c3-42d5-84b9-e96830929821 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1282.388221] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-50068599-b312-4f38-94bd-93ea358106d5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1282.397039] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf450f62-0cef-4fa5-8a4a-d0c81b5b0700 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1282.408986] env[67752]: DEBUG oslo_concurrency.lockutils [None req-50f8be6d-a6fa-472c-a580-8749396bef71 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "78e8ccdd-e5ed-4b87-af85-498ca755561b" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 224.349s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1282.418076] env[67752]: DEBUG nova.compute.manager [None req-9a19e1d7-beb1-47b1-89d6-4b4427bf93cb tempest-SecurityGroupsTestJSON-1418396030 tempest-SecurityGroupsTestJSON-1418396030-project-member] [instance: ffa5bab7-fce4-4fa7-9e43-c4fc35ee6c45] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1282.429290] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-d64f589f-61c3-42d5-84b9-e96830929821 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9 could not be found. [ 1282.429480] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d64f589f-61c3-42d5-84b9-e96830929821 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1282.429658] env[67752]: INFO nova.compute.manager [None req-d64f589f-61c3-42d5-84b9-e96830929821 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1282.429934] env[67752]: DEBUG oslo.service.loopingcall [None req-d64f589f-61c3-42d5-84b9-e96830929821 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1282.430345] env[67752]: DEBUG nova.compute.manager [-] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1282.430444] env[67752]: DEBUG nova.network.neutron [-] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1282.443028] env[67752]: DEBUG nova.compute.manager [None req-9a19e1d7-beb1-47b1-89d6-4b4427bf93cb tempest-SecurityGroupsTestJSON-1418396030 tempest-SecurityGroupsTestJSON-1418396030-project-member] [instance: ffa5bab7-fce4-4fa7-9e43-c4fc35ee6c45] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1282.448518] env[67752]: DEBUG nova.network.neutron [-] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1282.455672] env[67752]: DEBUG nova.network.neutron [-] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1282.463058] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a19e1d7-beb1-47b1-89d6-4b4427bf93cb tempest-SecurityGroupsTestJSON-1418396030 tempest-SecurityGroupsTestJSON-1418396030-project-member] Lock "ffa5bab7-fce4-4fa7-9e43-c4fc35ee6c45" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 222.951s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1282.464735] env[67752]: INFO nova.compute.manager [-] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] Took 0.03 seconds to deallocate network for instance. [ 1282.472902] env[67752]: DEBUG nova.compute.manager [None req-c3fd7bad-0c43-4856-bf43-3cbd4db3b6ee tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] [instance: 2573ebf9-e18d-4b07-bde6-459a2466fe0d] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1282.496273] env[67752]: DEBUG nova.compute.manager [None req-c3fd7bad-0c43-4856-bf43-3cbd4db3b6ee tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] [instance: 2573ebf9-e18d-4b07-bde6-459a2466fe0d] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1282.522224] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c3fd7bad-0c43-4856-bf43-3cbd4db3b6ee tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] Lock "2573ebf9-e18d-4b07-bde6-459a2466fe0d" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 221.520s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1282.531487] env[67752]: DEBUG nova.compute.manager [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1282.580594] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d64f589f-61c3-42d5-84b9-e96830929821 tempest-ImagesNegativeTestJSON-855599376 tempest-ImagesNegativeTestJSON-855599376-project-member] Lock "580c2aa9-160e-40ec-a1e1-0bbaaf60bca9" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.440s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1282.582460] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "580c2aa9-160e-40ec-a1e1-0bbaaf60bca9" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 102.942s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1282.582460] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 580c2aa9-160e-40ec-a1e1-0bbaaf60bca9] During sync_power_state the instance has a pending task (deleting). Skip. [ 1282.582460] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "580c2aa9-160e-40ec-a1e1-0bbaaf60bca9" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1282.595123] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1282.596932] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1282.596932] env[67752]: INFO nova.compute.claims [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1282.843811] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bc73b263-afe8-4b23-a7a9-0d2bb5a92287 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1282.851547] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ea6761b-6c5b-4c13-9bdb-05525286ed0b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1282.882722] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-27825006-599a-433c-8af5-63a5d1b63323 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1282.889758] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-edfce253-7b0b-417b-94d1-648f624e6cc1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1282.903263] env[67752]: DEBUG nova.compute.provider_tree [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1282.913020] env[67752]: DEBUG nova.scheduler.client.report [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1282.925946] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.330s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1282.926462] env[67752]: DEBUG nova.compute.manager [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1282.959656] env[67752]: DEBUG nova.compute.utils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1282.961278] env[67752]: DEBUG nova.compute.manager [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1282.961489] env[67752]: DEBUG nova.network.neutron [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1282.970581] env[67752]: DEBUG nova.compute.manager [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1283.034307] env[67752]: DEBUG nova.compute.manager [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1283.043077] env[67752]: DEBUG nova.policy [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '1d449c1148a04a2585b31a86da3dc3ec', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e15c74bbc9e742b9b1ef3b3bef0207aa', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1283.063547] env[67752]: DEBUG nova.virt.hardware [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1283.063812] env[67752]: DEBUG nova.virt.hardware [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1283.063975] env[67752]: DEBUG nova.virt.hardware [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1283.064186] env[67752]: DEBUG nova.virt.hardware [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1283.064339] env[67752]: DEBUG nova.virt.hardware [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1283.064492] env[67752]: DEBUG nova.virt.hardware [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1283.064699] env[67752]: DEBUG nova.virt.hardware [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1283.064886] env[67752]: DEBUG nova.virt.hardware [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1283.065324] env[67752]: DEBUG nova.virt.hardware [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1283.065556] env[67752]: DEBUG nova.virt.hardware [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1283.065793] env[67752]: DEBUG nova.virt.hardware [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1283.067088] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b6ff099f-90bd-4744-bc7c-b2eb64166bba {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1283.076115] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d71445d5-1b7a-4154-8c64-9798b6c27edc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1283.389022] env[67752]: DEBUG nova.network.neutron [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Successfully created port: 8a7a164a-15ff-4c0e-8bef-b0f1c5ced602 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1284.561776] env[67752]: DEBUG nova.compute.manager [req-0509cd60-0edc-4ff1-9bd2-ee4a110a9e48 req-007112d9-8f83-4cba-ac78-5efec912b99e service nova] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Received event network-vif-plugged-8a7a164a-15ff-4c0e-8bef-b0f1c5ced602 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1284.562048] env[67752]: DEBUG oslo_concurrency.lockutils [req-0509cd60-0edc-4ff1-9bd2-ee4a110a9e48 req-007112d9-8f83-4cba-ac78-5efec912b99e service nova] Acquiring lock "9432cd04-c2df-428b-b31a-93d3ec0c9d1f-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1284.564137] env[67752]: DEBUG oslo_concurrency.lockutils [req-0509cd60-0edc-4ff1-9bd2-ee4a110a9e48 req-007112d9-8f83-4cba-ac78-5efec912b99e service nova] Lock "9432cd04-c2df-428b-b31a-93d3ec0c9d1f-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1284.564137] env[67752]: DEBUG oslo_concurrency.lockutils [req-0509cd60-0edc-4ff1-9bd2-ee4a110a9e48 req-007112d9-8f83-4cba-ac78-5efec912b99e service nova] Lock "9432cd04-c2df-428b-b31a-93d3ec0c9d1f-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1284.564137] env[67752]: DEBUG nova.compute.manager [req-0509cd60-0edc-4ff1-9bd2-ee4a110a9e48 req-007112d9-8f83-4cba-ac78-5efec912b99e service nova] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] No waiting events found dispatching network-vif-plugged-8a7a164a-15ff-4c0e-8bef-b0f1c5ced602 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1284.564137] env[67752]: WARNING nova.compute.manager [req-0509cd60-0edc-4ff1-9bd2-ee4a110a9e48 req-007112d9-8f83-4cba-ac78-5efec912b99e service nova] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Received unexpected event network-vif-plugged-8a7a164a-15ff-4c0e-8bef-b0f1c5ced602 for instance with vm_state building and task_state spawning. [ 1284.594823] env[67752]: DEBUG nova.network.neutron [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Successfully updated port: 8a7a164a-15ff-4c0e-8bef-b0f1c5ced602 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1284.606804] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Acquiring lock "refresh_cache-9432cd04-c2df-428b-b31a-93d3ec0c9d1f" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1284.606951] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Acquired lock "refresh_cache-9432cd04-c2df-428b-b31a-93d3ec0c9d1f" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1284.607116] env[67752]: DEBUG nova.network.neutron [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1284.673544] env[67752]: DEBUG nova.network.neutron [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1284.911965] env[67752]: DEBUG nova.network.neutron [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Updating instance_info_cache with network_info: [{"id": "8a7a164a-15ff-4c0e-8bef-b0f1c5ced602", "address": "fa:16:3e:6d:21:4f", "network": {"id": "dd3a71fb-77d6-4603-8e7d-fa92c989a6e2", "bridge": "br-int", "label": "tempest-AttachInterfacesV270Test-165472694-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e15c74bbc9e742b9b1ef3b3bef0207aa", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "537e0890-4fa2-4f2d-b74c-49933a4edf53", "external-id": "nsx-vlan-transportzone-82", "segmentation_id": 82, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap8a7a164a-15", "ovs_interfaceid": "8a7a164a-15ff-4c0e-8bef-b0f1c5ced602", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1284.925109] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Releasing lock "refresh_cache-9432cd04-c2df-428b-b31a-93d3ec0c9d1f" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1284.925412] env[67752]: DEBUG nova.compute.manager [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Instance network_info: |[{"id": "8a7a164a-15ff-4c0e-8bef-b0f1c5ced602", "address": "fa:16:3e:6d:21:4f", "network": {"id": "dd3a71fb-77d6-4603-8e7d-fa92c989a6e2", "bridge": "br-int", "label": "tempest-AttachInterfacesV270Test-165472694-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e15c74bbc9e742b9b1ef3b3bef0207aa", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "537e0890-4fa2-4f2d-b74c-49933a4edf53", "external-id": "nsx-vlan-transportzone-82", "segmentation_id": 82, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap8a7a164a-15", "ovs_interfaceid": "8a7a164a-15ff-4c0e-8bef-b0f1c5ced602", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1284.925824] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:6d:21:4f', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '537e0890-4fa2-4f2d-b74c-49933a4edf53', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '8a7a164a-15ff-4c0e-8bef-b0f1c5ced602', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1284.933707] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Creating folder: Project (e15c74bbc9e742b9b1ef3b3bef0207aa). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1284.934247] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-23932a12-78c4-4073-82c9-57fb14790ff9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1284.946586] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Created folder: Project (e15c74bbc9e742b9b1ef3b3bef0207aa) in parent group-v639722. [ 1284.946771] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Creating folder: Instances. Parent ref: group-v639806. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1284.946991] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-201b95d1-2d68-4e57-9f9d-6e7704c6b449 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1284.954962] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Created folder: Instances in parent group-v639806. [ 1284.955216] env[67752]: DEBUG oslo.service.loopingcall [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1284.955398] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1284.955623] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-8ae812bd-dcb7-45fa-b5c9-bdbcb37126ac {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1284.974099] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1284.974099] env[67752]: value = "task-3199763" [ 1284.974099] env[67752]: _type = "Task" [ 1284.974099] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1284.981297] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199763, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1285.484647] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199763, 'name': CreateVM_Task, 'duration_secs': 0.28354} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1285.484819] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1285.485539] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1285.485716] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1285.486067] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1285.486347] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-4e0f241e-3eb3-4f02-8af5-5612192b055d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1285.490999] env[67752]: DEBUG oslo_vmware.api [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Waiting for the task: (returnval){ [ 1285.490999] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52778db4-bd3e-09c1-de72-382e254fdb51" [ 1285.490999] env[67752]: _type = "Task" [ 1285.490999] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1285.498303] env[67752]: DEBUG oslo_vmware.api [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52778db4-bd3e-09c1-de72-382e254fdb51, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1286.001681] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1286.002046] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1286.002162] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1286.594396] env[67752]: DEBUG nova.compute.manager [req-c0bda136-8c20-4ab1-ac52-9b1461d5363a req-edb4a76f-467a-4f89-b266-26a216cb9c56 service nova] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Received event network-changed-8a7a164a-15ff-4c0e-8bef-b0f1c5ced602 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1286.594396] env[67752]: DEBUG nova.compute.manager [req-c0bda136-8c20-4ab1-ac52-9b1461d5363a req-edb4a76f-467a-4f89-b266-26a216cb9c56 service nova] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Refreshing instance network info cache due to event network-changed-8a7a164a-15ff-4c0e-8bef-b0f1c5ced602. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1286.594396] env[67752]: DEBUG oslo_concurrency.lockutils [req-c0bda136-8c20-4ab1-ac52-9b1461d5363a req-edb4a76f-467a-4f89-b266-26a216cb9c56 service nova] Acquiring lock "refresh_cache-9432cd04-c2df-428b-b31a-93d3ec0c9d1f" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1286.594518] env[67752]: DEBUG oslo_concurrency.lockutils [req-c0bda136-8c20-4ab1-ac52-9b1461d5363a req-edb4a76f-467a-4f89-b266-26a216cb9c56 service nova] Acquired lock "refresh_cache-9432cd04-c2df-428b-b31a-93d3ec0c9d1f" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1286.594669] env[67752]: DEBUG nova.network.neutron [req-c0bda136-8c20-4ab1-ac52-9b1461d5363a req-edb4a76f-467a-4f89-b266-26a216cb9c56 service nova] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Refreshing network info cache for port 8a7a164a-15ff-4c0e-8bef-b0f1c5ced602 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1286.969559] env[67752]: DEBUG nova.network.neutron [req-c0bda136-8c20-4ab1-ac52-9b1461d5363a req-edb4a76f-467a-4f89-b266-26a216cb9c56 service nova] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Updated VIF entry in instance network info cache for port 8a7a164a-15ff-4c0e-8bef-b0f1c5ced602. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1286.969960] env[67752]: DEBUG nova.network.neutron [req-c0bda136-8c20-4ab1-ac52-9b1461d5363a req-edb4a76f-467a-4f89-b266-26a216cb9c56 service nova] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Updating instance_info_cache with network_info: [{"id": "8a7a164a-15ff-4c0e-8bef-b0f1c5ced602", "address": "fa:16:3e:6d:21:4f", "network": {"id": "dd3a71fb-77d6-4603-8e7d-fa92c989a6e2", "bridge": "br-int", "label": "tempest-AttachInterfacesV270Test-165472694-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e15c74bbc9e742b9b1ef3b3bef0207aa", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "537e0890-4fa2-4f2d-b74c-49933a4edf53", "external-id": "nsx-vlan-transportzone-82", "segmentation_id": 82, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap8a7a164a-15", "ovs_interfaceid": "8a7a164a-15ff-4c0e-8bef-b0f1c5ced602", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1286.979350] env[67752]: DEBUG oslo_concurrency.lockutils [req-c0bda136-8c20-4ab1-ac52-9b1461d5363a req-edb4a76f-467a-4f89-b266-26a216cb9c56 service nova] Releasing lock "refresh_cache-9432cd04-c2df-428b-b31a-93d3ec0c9d1f" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1288.061201] env[67752]: DEBUG oslo_concurrency.lockutils [None req-79213572-3c2a-410f-a3f0-655d520da73e tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Acquiring lock "9432cd04-c2df-428b-b31a-93d3ec0c9d1f" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1318.631634] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1322.634039] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1322.634382] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 1322.634382] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 1322.658953] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1322.659153] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1322.659283] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1322.659419] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1322.659548] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1322.659674] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1322.659798] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: d3940c29-852a-427b-9027-aa4080150724] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1322.659922] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1322.660061] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1322.660189] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1322.660350] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 1322.660890] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1322.661084] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1323.635490] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1324.634631] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1324.634872] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1324.635044] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 1327.164442] env[67752]: WARNING oslo_vmware.rw_handles [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1327.164442] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1327.164442] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1327.164442] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1327.164442] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1327.164442] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1327.164442] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1327.164442] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1327.164442] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1327.164442] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1327.164442] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1327.164442] env[67752]: ERROR oslo_vmware.rw_handles [ 1327.164947] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/1d5bd875-f8d8-4738-b397-7ce1c9799909/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1327.167007] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1327.167270] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Copying Virtual Disk [datastore2] vmware_temp/1d5bd875-f8d8-4738-b397-7ce1c9799909/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/1d5bd875-f8d8-4738-b397-7ce1c9799909/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1327.167553] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-c9255491-0a0f-4e84-a96f-691bf59e4e15 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1327.175375] env[67752]: DEBUG oslo_vmware.api [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Waiting for the task: (returnval){ [ 1327.175375] env[67752]: value = "task-3199764" [ 1327.175375] env[67752]: _type = "Task" [ 1327.175375] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1327.183529] env[67752]: DEBUG oslo_vmware.api [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Task: {'id': task-3199764, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1327.634608] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1327.647061] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1327.647312] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1327.647491] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1327.647815] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1327.648859] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-afe7641d-e864-4f61-a159-fe278350021a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1327.657716] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac4501a7-38a6-4022-99bd-44be9df4cf19 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1327.671727] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8cdd7532-b142-479d-9a93-9dd63579a06b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1327.680644] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b9ff8444-aa3b-403a-a801-1a386b850eb4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1327.688544] env[67752]: DEBUG oslo_vmware.exceptions [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1327.711782] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1327.712362] env[67752]: ERROR nova.compute.manager [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1327.712362] env[67752]: Faults: ['InvalidArgument'] [ 1327.712362] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Traceback (most recent call last): [ 1327.712362] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1327.712362] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] yield resources [ 1327.712362] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1327.712362] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] self.driver.spawn(context, instance, image_meta, [ 1327.712362] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1327.712362] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1327.712362] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1327.712362] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] self._fetch_image_if_missing(context, vi) [ 1327.712362] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1327.712674] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] image_cache(vi, tmp_image_ds_loc) [ 1327.712674] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1327.712674] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] vm_util.copy_virtual_disk( [ 1327.712674] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1327.712674] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] session._wait_for_task(vmdk_copy_task) [ 1327.712674] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1327.712674] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] return self.wait_for_task(task_ref) [ 1327.712674] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1327.712674] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] return evt.wait() [ 1327.712674] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1327.712674] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] result = hub.switch() [ 1327.712674] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1327.712674] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] return self.greenlet.switch() [ 1327.712970] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1327.712970] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] self.f(*self.args, **self.kw) [ 1327.712970] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1327.712970] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] raise exceptions.translate_fault(task_info.error) [ 1327.712970] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1327.712970] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Faults: ['InvalidArgument'] [ 1327.712970] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] [ 1327.712970] env[67752]: INFO nova.compute.manager [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Terminating instance [ 1327.715390] env[67752]: DEBUG nova.compute.manager [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1327.715590] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1327.715928] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180961MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1327.716079] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1327.716271] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1327.718223] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1327.718558] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1327.719309] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ec60f45-6aae-449f-a61c-2b04e09d338b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1327.722018] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-68bfa815-9b3c-428e-99ea-84d738b27c4a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1327.729241] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1327.729457] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-88dd1a19-97ce-4ff9-af6c-faa67abed59a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1327.731984] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1327.732241] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1327.732830] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-d9e04799-b8e4-46a3-9efc-7ecb0f6d7273 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1327.737254] env[67752]: DEBUG oslo_vmware.api [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Waiting for the task: (returnval){ [ 1327.737254] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52cc90f4-1b75-2d67-0a1c-37b69aaf41f8" [ 1327.737254] env[67752]: _type = "Task" [ 1327.737254] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1327.744686] env[67752]: DEBUG oslo_vmware.api [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52cc90f4-1b75-2d67-0a1c-37b69aaf41f8, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1327.791483] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 411bafb7-ef72-4529-b3f3-0b4955f23788 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1327.791676] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ada70883-9bef-496e-a65f-ca84f6ee6211 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1327.791828] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1327.792009] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b271cd73-a148-48d6-bed9-6a99512457bd actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1327.792155] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1327.792306] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f6499ee5-cb24-44e1-9fe7-c19feeb732eb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1327.792493] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d3940c29-852a-427b-9027-aa4080150724 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1327.792562] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1327.792634] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance a8bd310e-530a-46ab-add7-1b827ea5f399 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1327.792733] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9432cd04-c2df-428b-b31a-93d3ec0c9d1f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1327.795173] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1327.795384] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1327.795566] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Deleting the datastore file [datastore2] 411bafb7-ef72-4529-b3f3-0b4955f23788 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1327.795828] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-ef0065aa-d9d2-48b7-8dba-7f1034c59c1e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1327.803246] env[67752]: DEBUG oslo_vmware.api [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Waiting for the task: (returnval){ [ 1327.803246] env[67752]: value = "task-3199766" [ 1327.803246] env[67752]: _type = "Task" [ 1327.803246] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1327.811195] env[67752]: DEBUG oslo_vmware.api [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Task: {'id': task-3199766, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1327.829758] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 90e58033-eacd-433a-8d25-40f7d8403588 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1327.841629] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 81738fe6-6d50-4dfe-ae86-4c8771837168 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1327.853120] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e4ebb0f0-ab58-491f-bc4a-7548bbe415d8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1327.866511] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ba83b143-d752-4166-8ce0-fbd7fd4875a4 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1327.877016] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 6000d3ce-6bdc-4683-a1a9-fbc1e447aad4 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1327.887103] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fe397d83-8625-4c5c-88b2-6c96c51c42bf has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1327.896546] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9ce8f114-c06d-4f08-b64a-980b94975ff7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1327.905757] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 5142518d-3b2a-474c-81d9-d658cb8467e3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1327.905995] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1327.906168] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1328.097048] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-98c3b5a2-b23d-4fd2-9196-b34c1d04668f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1328.104511] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bfd4f3e0-1143-4dbb-aaa1-e599563abc65 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1328.134463] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-14f39079-b2d7-4c8e-91ff-d8fd58575296 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1328.141171] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e8fd9da-5262-4e61-a641-764c6cebc59e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1328.154408] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1328.162517] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1328.179528] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1328.179769] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.463s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1328.247904] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1328.248167] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Creating directory with path [datastore2] vmware_temp/261fdf91-bfc9-42a0-99e2-7b94758afecc/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1328.248411] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-4bbccbbb-a519-427d-b88a-27ec9eca8f05 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1328.258855] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Created directory with path [datastore2] vmware_temp/261fdf91-bfc9-42a0-99e2-7b94758afecc/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1328.259050] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Fetch image to [datastore2] vmware_temp/261fdf91-bfc9-42a0-99e2-7b94758afecc/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1328.259228] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/261fdf91-bfc9-42a0-99e2-7b94758afecc/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1328.259902] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-788c18e8-79da-42a0-95be-cd39207a6945 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1328.266233] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2b9bf46d-6778-46ad-8145-698e78bda537 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1328.274886] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-76326d20-0121-467b-9503-6d4e0e4dadb0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1328.306763] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-145de78f-973f-4198-9116-4b5aa012ea6b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1328.314784] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-fd248117-3915-4911-ba58-1bbfdf29dc92 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1328.316361] env[67752]: DEBUG oslo_vmware.api [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Task: {'id': task-3199766, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.067904} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1328.316591] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1328.316772] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1328.316946] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1328.317134] env[67752]: INFO nova.compute.manager [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1328.319238] env[67752]: DEBUG nova.compute.claims [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1328.319408] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1328.319623] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1328.340780] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1328.396243] env[67752]: DEBUG oslo_vmware.rw_handles [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/261fdf91-bfc9-42a0-99e2-7b94758afecc/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1328.458671] env[67752]: DEBUG oslo_vmware.rw_handles [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1328.458941] env[67752]: DEBUG oslo_vmware.rw_handles [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Closing write handle for https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/261fdf91-bfc9-42a0-99e2-7b94758afecc/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1328.636073] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d422532a-587e-41d3-8608-2bc716808865 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1328.643971] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e73b80af-631a-4c45-8a57-9c3b4b0c7fdf {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1328.674974] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0cb97732-90f6-4f7e-9f5c-827c045f988a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1328.682512] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44da2e65-0710-4279-917f-832f6a256ed8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1328.695609] env[67752]: DEBUG nova.compute.provider_tree [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1328.705151] env[67752]: DEBUG nova.scheduler.client.report [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1328.718873] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.399s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1328.719430] env[67752]: ERROR nova.compute.manager [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1328.719430] env[67752]: Faults: ['InvalidArgument'] [ 1328.719430] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Traceback (most recent call last): [ 1328.719430] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1328.719430] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] self.driver.spawn(context, instance, image_meta, [ 1328.719430] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1328.719430] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1328.719430] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1328.719430] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] self._fetch_image_if_missing(context, vi) [ 1328.719430] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1328.719430] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] image_cache(vi, tmp_image_ds_loc) [ 1328.719430] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1328.719718] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] vm_util.copy_virtual_disk( [ 1328.719718] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1328.719718] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] session._wait_for_task(vmdk_copy_task) [ 1328.719718] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1328.719718] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] return self.wait_for_task(task_ref) [ 1328.719718] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1328.719718] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] return evt.wait() [ 1328.719718] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1328.719718] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] result = hub.switch() [ 1328.719718] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1328.719718] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] return self.greenlet.switch() [ 1328.719718] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1328.719718] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] self.f(*self.args, **self.kw) [ 1328.719995] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1328.719995] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] raise exceptions.translate_fault(task_info.error) [ 1328.719995] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1328.719995] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Faults: ['InvalidArgument'] [ 1328.719995] env[67752]: ERROR nova.compute.manager [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] [ 1328.720126] env[67752]: DEBUG nova.compute.utils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1328.721888] env[67752]: DEBUG nova.compute.manager [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Build of instance 411bafb7-ef72-4529-b3f3-0b4955f23788 was re-scheduled: A specified parameter was not correct: fileType [ 1328.721888] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1328.722270] env[67752]: DEBUG nova.compute.manager [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1328.722444] env[67752]: DEBUG nova.compute.manager [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1328.722626] env[67752]: DEBUG nova.compute.manager [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1328.722790] env[67752]: DEBUG nova.network.neutron [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1329.231294] env[67752]: DEBUG nova.network.neutron [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1329.248567] env[67752]: INFO nova.compute.manager [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Took 0.53 seconds to deallocate network for instance. [ 1329.367025] env[67752]: INFO nova.scheduler.client.report [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Deleted allocations for instance 411bafb7-ef72-4529-b3f3-0b4955f23788 [ 1329.392253] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a5a67e10-df23-462a-b09d-a0a5ab1c4fa9 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Lock "411bafb7-ef72-4529-b3f3-0b4955f23788" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 636.555s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1329.393369] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e3a1bc7e-3511-4319-bcee-07db92c8e9a1 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Lock "411bafb7-ef72-4529-b3f3-0b4955f23788" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 440.162s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1329.393592] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e3a1bc7e-3511-4319-bcee-07db92c8e9a1 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Acquiring lock "411bafb7-ef72-4529-b3f3-0b4955f23788-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1329.393796] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e3a1bc7e-3511-4319-bcee-07db92c8e9a1 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Lock "411bafb7-ef72-4529-b3f3-0b4955f23788-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1329.393961] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e3a1bc7e-3511-4319-bcee-07db92c8e9a1 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Lock "411bafb7-ef72-4529-b3f3-0b4955f23788-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1329.396035] env[67752]: INFO nova.compute.manager [None req-e3a1bc7e-3511-4319-bcee-07db92c8e9a1 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Terminating instance [ 1329.397536] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e3a1bc7e-3511-4319-bcee-07db92c8e9a1 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Acquiring lock "refresh_cache-411bafb7-ef72-4529-b3f3-0b4955f23788" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1329.397727] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e3a1bc7e-3511-4319-bcee-07db92c8e9a1 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Acquired lock "refresh_cache-411bafb7-ef72-4529-b3f3-0b4955f23788" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1329.397900] env[67752]: DEBUG nova.network.neutron [None req-e3a1bc7e-3511-4319-bcee-07db92c8e9a1 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1329.404824] env[67752]: DEBUG nova.compute.manager [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1329.425480] env[67752]: DEBUG nova.network.neutron [None req-e3a1bc7e-3511-4319-bcee-07db92c8e9a1 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1329.458125] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1329.458396] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1329.459887] env[67752]: INFO nova.compute.claims [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1329.576142] env[67752]: DEBUG nova.network.neutron [None req-e3a1bc7e-3511-4319-bcee-07db92c8e9a1 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1329.589251] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e3a1bc7e-3511-4319-bcee-07db92c8e9a1 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Releasing lock "refresh_cache-411bafb7-ef72-4529-b3f3-0b4955f23788" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1329.589903] env[67752]: DEBUG nova.compute.manager [None req-e3a1bc7e-3511-4319-bcee-07db92c8e9a1 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1329.590234] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-e3a1bc7e-3511-4319-bcee-07db92c8e9a1 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1329.590981] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-2242879f-55f0-4576-abc1-1da56aa9c249 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1329.603645] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6c67c851-8d5c-4aa3-a0ac-5f7983c66f4a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1329.637076] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-e3a1bc7e-3511-4319-bcee-07db92c8e9a1 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 411bafb7-ef72-4529-b3f3-0b4955f23788 could not be found. [ 1329.637303] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-e3a1bc7e-3511-4319-bcee-07db92c8e9a1 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1329.637488] env[67752]: INFO nova.compute.manager [None req-e3a1bc7e-3511-4319-bcee-07db92c8e9a1 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1329.637736] env[67752]: DEBUG oslo.service.loopingcall [None req-e3a1bc7e-3511-4319-bcee-07db92c8e9a1 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1329.640302] env[67752]: DEBUG nova.compute.manager [-] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1329.640425] env[67752]: DEBUG nova.network.neutron [-] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1329.656627] env[67752]: DEBUG nova.network.neutron [-] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1329.664090] env[67752]: DEBUG nova.network.neutron [-] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1329.674783] env[67752]: INFO nova.compute.manager [-] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] Took 0.03 seconds to deallocate network for instance. [ 1329.731803] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-002490b1-6a08-4519-a178-5da3cc73ecf9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1329.739256] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90f7e55d-d45b-464f-b704-fb81ba772c3c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1329.770486] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-322100a4-482d-41aa-88b4-d4fad12dd369 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1329.775318] env[67752]: DEBUG oslo_concurrency.lockutils [None req-e3a1bc7e-3511-4319-bcee-07db92c8e9a1 tempest-FloatingIPsAssociationTestJSON-2055802534 tempest-FloatingIPsAssociationTestJSON-2055802534-project-member] Lock "411bafb7-ef72-4529-b3f3-0b4955f23788" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.382s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1329.776487] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "411bafb7-ef72-4529-b3f3-0b4955f23788" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 150.136s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1329.776682] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 411bafb7-ef72-4529-b3f3-0b4955f23788] During sync_power_state the instance has a pending task (deleting). Skip. [ 1329.776857] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "411bafb7-ef72-4529-b3f3-0b4955f23788" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1329.780548] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d526a0ba-1aea-4959-b5f8-434237ded67c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1329.794320] env[67752]: DEBUG nova.compute.provider_tree [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1329.802122] env[67752]: DEBUG nova.scheduler.client.report [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1329.817130] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.358s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1329.817130] env[67752]: DEBUG nova.compute.manager [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1329.849387] env[67752]: DEBUG nova.compute.utils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1329.850910] env[67752]: DEBUG nova.compute.manager [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1329.851135] env[67752]: DEBUG nova.network.neutron [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1329.863876] env[67752]: DEBUG nova.compute.manager [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1329.931243] env[67752]: DEBUG nova.compute.manager [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1329.959874] env[67752]: DEBUG nova.virt.hardware [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1329.960091] env[67752]: DEBUG nova.virt.hardware [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1329.960254] env[67752]: DEBUG nova.virt.hardware [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1329.960506] env[67752]: DEBUG nova.virt.hardware [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1329.960668] env[67752]: DEBUG nova.virt.hardware [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1329.960889] env[67752]: DEBUG nova.virt.hardware [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1329.961037] env[67752]: DEBUG nova.virt.hardware [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1329.961207] env[67752]: DEBUG nova.virt.hardware [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1329.961396] env[67752]: DEBUG nova.virt.hardware [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1329.961579] env[67752]: DEBUG nova.virt.hardware [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1329.961757] env[67752]: DEBUG nova.virt.hardware [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1329.962834] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-18d5ef85-426a-485e-8304-59e3f70b8081 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1329.970968] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f17dc4d6-0cd0-4742-80af-9a90168b8583 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1329.986667] env[67752]: DEBUG nova.policy [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7c16a07a972f418fb0d312dae117f843', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '3eaa4b9c93d94830b669e0c56d36afd7', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1330.179857] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1330.421642] env[67752]: DEBUG nova.network.neutron [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Successfully created port: 06ea4893-c92f-43e8-b693-e223228aaf67 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1331.330778] env[67752]: DEBUG nova.network.neutron [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Successfully updated port: 06ea4893-c92f-43e8-b693-e223228aaf67 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1331.337358] env[67752]: DEBUG nova.compute.manager [req-e17ca2af-b824-409c-9ec1-269f9a08b7e4 req-7346da84-a261-4813-a44d-372539cc3a49 service nova] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Received event network-vif-plugged-06ea4893-c92f-43e8-b693-e223228aaf67 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1331.337358] env[67752]: DEBUG oslo_concurrency.lockutils [req-e17ca2af-b824-409c-9ec1-269f9a08b7e4 req-7346da84-a261-4813-a44d-372539cc3a49 service nova] Acquiring lock "90e58033-eacd-433a-8d25-40f7d8403588-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1331.337358] env[67752]: DEBUG oslo_concurrency.lockutils [req-e17ca2af-b824-409c-9ec1-269f9a08b7e4 req-7346da84-a261-4813-a44d-372539cc3a49 service nova] Lock "90e58033-eacd-433a-8d25-40f7d8403588-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1331.337358] env[67752]: DEBUG oslo_concurrency.lockutils [req-e17ca2af-b824-409c-9ec1-269f9a08b7e4 req-7346da84-a261-4813-a44d-372539cc3a49 service nova] Lock "90e58033-eacd-433a-8d25-40f7d8403588-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1331.337546] env[67752]: DEBUG nova.compute.manager [req-e17ca2af-b824-409c-9ec1-269f9a08b7e4 req-7346da84-a261-4813-a44d-372539cc3a49 service nova] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] No waiting events found dispatching network-vif-plugged-06ea4893-c92f-43e8-b693-e223228aaf67 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1331.337546] env[67752]: WARNING nova.compute.manager [req-e17ca2af-b824-409c-9ec1-269f9a08b7e4 req-7346da84-a261-4813-a44d-372539cc3a49 service nova] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Received unexpected event network-vif-plugged-06ea4893-c92f-43e8-b693-e223228aaf67 for instance with vm_state building and task_state spawning. [ 1331.351708] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Acquiring lock "refresh_cache-90e58033-eacd-433a-8d25-40f7d8403588" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1331.352220] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Acquired lock "refresh_cache-90e58033-eacd-433a-8d25-40f7d8403588" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1331.353194] env[67752]: DEBUG nova.network.neutron [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1331.431501] env[67752]: DEBUG nova.network.neutron [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1331.980779] env[67752]: DEBUG nova.network.neutron [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Updating instance_info_cache with network_info: [{"id": "06ea4893-c92f-43e8-b693-e223228aaf67", "address": "fa:16:3e:5c:73:ae", "network": {"id": "134ea55d-2bdd-4330-a123-873dbcfa8f5f", "bridge": "br-int", "label": "tempest-ServerAddressesNegativeTestJSON-71180782-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3eaa4b9c93d94830b669e0c56d36afd7", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "6815237d-f565-474d-a3c0-9c675478eb00", "external-id": "nsx-vlan-transportzone-526", "segmentation_id": 526, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap06ea4893-c9", "ovs_interfaceid": "06ea4893-c92f-43e8-b693-e223228aaf67", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1331.994258] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Releasing lock "refresh_cache-90e58033-eacd-433a-8d25-40f7d8403588" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1331.994549] env[67752]: DEBUG nova.compute.manager [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Instance network_info: |[{"id": "06ea4893-c92f-43e8-b693-e223228aaf67", "address": "fa:16:3e:5c:73:ae", "network": {"id": "134ea55d-2bdd-4330-a123-873dbcfa8f5f", "bridge": "br-int", "label": "tempest-ServerAddressesNegativeTestJSON-71180782-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3eaa4b9c93d94830b669e0c56d36afd7", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "6815237d-f565-474d-a3c0-9c675478eb00", "external-id": "nsx-vlan-transportzone-526", "segmentation_id": 526, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap06ea4893-c9", "ovs_interfaceid": "06ea4893-c92f-43e8-b693-e223228aaf67", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1331.994922] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:5c:73:ae', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '6815237d-f565-474d-a3c0-9c675478eb00', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '06ea4893-c92f-43e8-b693-e223228aaf67', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1332.002628] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Creating folder: Project (3eaa4b9c93d94830b669e0c56d36afd7). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1332.003183] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-c66d6f05-705f-4b1b-8dba-ffae839cc9f5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1332.013087] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Created folder: Project (3eaa4b9c93d94830b669e0c56d36afd7) in parent group-v639722. [ 1332.013281] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Creating folder: Instances. Parent ref: group-v639809. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1332.013498] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-7fb7d65e-e905-4ae9-8dbf-ed4d24880eac {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1332.023468] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Created folder: Instances in parent group-v639809. [ 1332.023725] env[67752]: DEBUG oslo.service.loopingcall [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1332.023907] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1332.024117] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-4601035a-faad-40c8-a874-2b40d36e7619 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1332.042475] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1332.042475] env[67752]: value = "task-3199769" [ 1332.042475] env[67752]: _type = "Task" [ 1332.042475] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1332.049995] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199769, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1332.554890] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199769, 'name': CreateVM_Task, 'duration_secs': 0.280933} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1332.554890] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1332.554890] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1332.554890] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1332.554890] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1332.555444] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-d3cc946d-813c-4e1a-bd0a-5a7dff0e76e9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1332.558230] env[67752]: DEBUG oslo_vmware.api [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Waiting for the task: (returnval){ [ 1332.558230] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]521ef7dc-f1a6-770b-6f83-c7e4add1bbca" [ 1332.558230] env[67752]: _type = "Task" [ 1332.558230] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1332.565976] env[67752]: DEBUG oslo_vmware.api [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]521ef7dc-f1a6-770b-6f83-c7e4add1bbca, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1333.076246] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1333.076246] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1333.076246] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1333.374453] env[67752]: DEBUG nova.compute.manager [req-e29c7c56-f622-4dee-b422-09a7c0388d43 req-f5a76089-7672-4b70-9b99-2087b3bc39f4 service nova] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Received event network-changed-06ea4893-c92f-43e8-b693-e223228aaf67 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1333.374639] env[67752]: DEBUG nova.compute.manager [req-e29c7c56-f622-4dee-b422-09a7c0388d43 req-f5a76089-7672-4b70-9b99-2087b3bc39f4 service nova] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Refreshing instance network info cache due to event network-changed-06ea4893-c92f-43e8-b693-e223228aaf67. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1333.374855] env[67752]: DEBUG oslo_concurrency.lockutils [req-e29c7c56-f622-4dee-b422-09a7c0388d43 req-f5a76089-7672-4b70-9b99-2087b3bc39f4 service nova] Acquiring lock "refresh_cache-90e58033-eacd-433a-8d25-40f7d8403588" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1333.375008] env[67752]: DEBUG oslo_concurrency.lockutils [req-e29c7c56-f622-4dee-b422-09a7c0388d43 req-f5a76089-7672-4b70-9b99-2087b3bc39f4 service nova] Acquired lock "refresh_cache-90e58033-eacd-433a-8d25-40f7d8403588" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1333.375195] env[67752]: DEBUG nova.network.neutron [req-e29c7c56-f622-4dee-b422-09a7c0388d43 req-f5a76089-7672-4b70-9b99-2087b3bc39f4 service nova] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Refreshing network info cache for port 06ea4893-c92f-43e8-b693-e223228aaf67 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1334.125415] env[67752]: DEBUG nova.network.neutron [req-e29c7c56-f622-4dee-b422-09a7c0388d43 req-f5a76089-7672-4b70-9b99-2087b3bc39f4 service nova] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Updated VIF entry in instance network info cache for port 06ea4893-c92f-43e8-b693-e223228aaf67. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1334.127319] env[67752]: DEBUG nova.network.neutron [req-e29c7c56-f622-4dee-b422-09a7c0388d43 req-f5a76089-7672-4b70-9b99-2087b3bc39f4 service nova] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Updating instance_info_cache with network_info: [{"id": "06ea4893-c92f-43e8-b693-e223228aaf67", "address": "fa:16:3e:5c:73:ae", "network": {"id": "134ea55d-2bdd-4330-a123-873dbcfa8f5f", "bridge": "br-int", "label": "tempest-ServerAddressesNegativeTestJSON-71180782-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "3eaa4b9c93d94830b669e0c56d36afd7", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "6815237d-f565-474d-a3c0-9c675478eb00", "external-id": "nsx-vlan-transportzone-526", "segmentation_id": 526, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap06ea4893-c9", "ovs_interfaceid": "06ea4893-c92f-43e8-b693-e223228aaf67", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1334.137754] env[67752]: DEBUG oslo_concurrency.lockutils [req-e29c7c56-f622-4dee-b422-09a7c0388d43 req-f5a76089-7672-4b70-9b99-2087b3bc39f4 service nova] Releasing lock "refresh_cache-90e58033-eacd-433a-8d25-40f7d8403588" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1342.931757] env[67752]: DEBUG oslo_concurrency.lockutils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquiring lock "c1975c17-15a5-4751-bf7f-d5b7e0d2fea1" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1342.931757] env[67752]: DEBUG oslo_concurrency.lockutils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "c1975c17-15a5-4751-bf7f-d5b7e0d2fea1" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1374.382080] env[67752]: WARNING oslo_vmware.rw_handles [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1374.382080] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1374.382080] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1374.382080] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1374.382080] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1374.382080] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1374.382080] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1374.382080] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1374.382080] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1374.382080] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1374.382080] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1374.382080] env[67752]: ERROR oslo_vmware.rw_handles [ 1374.382742] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/261fdf91-bfc9-42a0-99e2-7b94758afecc/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1374.384471] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1374.384721] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Copying Virtual Disk [datastore2] vmware_temp/261fdf91-bfc9-42a0-99e2-7b94758afecc/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/261fdf91-bfc9-42a0-99e2-7b94758afecc/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1374.385018] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-4aaeb136-e1d5-41ea-bb66-531c90d58761 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1374.392527] env[67752]: DEBUG oslo_vmware.api [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Waiting for the task: (returnval){ [ 1374.392527] env[67752]: value = "task-3199770" [ 1374.392527] env[67752]: _type = "Task" [ 1374.392527] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1374.400339] env[67752]: DEBUG oslo_vmware.api [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Task: {'id': task-3199770, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1374.903440] env[67752]: DEBUG oslo_vmware.exceptions [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1374.903572] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1374.904139] env[67752]: ERROR nova.compute.manager [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1374.904139] env[67752]: Faults: ['InvalidArgument'] [ 1374.904139] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Traceback (most recent call last): [ 1374.904139] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1374.904139] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] yield resources [ 1374.904139] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1374.904139] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] self.driver.spawn(context, instance, image_meta, [ 1374.904139] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1374.904139] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1374.904139] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1374.904139] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] self._fetch_image_if_missing(context, vi) [ 1374.904139] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1374.904511] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] image_cache(vi, tmp_image_ds_loc) [ 1374.904511] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1374.904511] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] vm_util.copy_virtual_disk( [ 1374.904511] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1374.904511] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] session._wait_for_task(vmdk_copy_task) [ 1374.904511] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1374.904511] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] return self.wait_for_task(task_ref) [ 1374.904511] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1374.904511] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] return evt.wait() [ 1374.904511] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1374.904511] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] result = hub.switch() [ 1374.904511] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1374.904511] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] return self.greenlet.switch() [ 1374.905000] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1374.905000] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] self.f(*self.args, **self.kw) [ 1374.905000] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1374.905000] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] raise exceptions.translate_fault(task_info.error) [ 1374.905000] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1374.905000] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Faults: ['InvalidArgument'] [ 1374.905000] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] [ 1374.905000] env[67752]: INFO nova.compute.manager [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Terminating instance [ 1374.906012] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1374.906289] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1374.906473] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-58c03d2f-79df-4f89-8171-3540580ca713 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1374.908885] env[67752]: DEBUG nova.compute.manager [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1374.909097] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1374.909810] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dc2bc0fe-6eaf-487a-ba5c-4e4db0677282 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1374.916369] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1374.916565] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-26eb6c74-915d-4ebe-a92f-a392b9d61d8b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1374.918737] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1374.919058] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1374.919724] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-16336b66-3cc2-4d35-853a-8be15a4d7905 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1374.924461] env[67752]: DEBUG oslo_vmware.api [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Waiting for the task: (returnval){ [ 1374.924461] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]528e4e42-1338-6c1c-2621-0e7ef437f65f" [ 1374.924461] env[67752]: _type = "Task" [ 1374.924461] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1374.931292] env[67752]: DEBUG oslo_vmware.api [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]528e4e42-1338-6c1c-2621-0e7ef437f65f, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1374.986741] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1374.986977] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1374.987178] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Deleting the datastore file [datastore2] ada70883-9bef-496e-a65f-ca84f6ee6211 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1374.987447] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-ade7da2c-22bb-451e-a528-0312a89daa15 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1374.993661] env[67752]: DEBUG oslo_vmware.api [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Waiting for the task: (returnval){ [ 1374.993661] env[67752]: value = "task-3199772" [ 1374.993661] env[67752]: _type = "Task" [ 1374.993661] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1375.000951] env[67752]: DEBUG oslo_vmware.api [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Task: {'id': task-3199772, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1375.435592] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1375.435913] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Creating directory with path [datastore2] vmware_temp/05471069-e7c2-401f-8c70-9b70fbcc167b/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1375.436107] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-ba3fe18a-7bff-4823-89b1-23ae5e64e510 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1375.447359] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Created directory with path [datastore2] vmware_temp/05471069-e7c2-401f-8c70-9b70fbcc167b/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1375.447549] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Fetch image to [datastore2] vmware_temp/05471069-e7c2-401f-8c70-9b70fbcc167b/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1375.447718] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/05471069-e7c2-401f-8c70-9b70fbcc167b/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1375.448448] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ee30ee2-f75d-48ad-8a8a-8cdbcca835b8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1375.454988] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9bf2000f-1961-48c9-b0f5-184ef5ad1c52 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1375.463772] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6747ec5a-b9fd-42ff-a956-85d89222d7e1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1375.494281] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cbb96206-0fa5-44d6-9369-247eb8471be8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1375.504656] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-24964d4e-be4b-4cea-b3a1-43171de849e6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1375.506288] env[67752]: DEBUG oslo_vmware.api [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Task: {'id': task-3199772, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.078315} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1375.506556] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1375.506738] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1375.506907] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1375.507095] env[67752]: INFO nova.compute.manager [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1375.509176] env[67752]: DEBUG nova.compute.claims [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1375.509357] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1375.509574] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1375.528330] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1375.581542] env[67752]: DEBUG oslo_vmware.rw_handles [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/05471069-e7c2-401f-8c70-9b70fbcc167b/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1375.643561] env[67752]: DEBUG oslo_vmware.rw_handles [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1375.643748] env[67752]: DEBUG oslo_vmware.rw_handles [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/05471069-e7c2-401f-8c70-9b70fbcc167b/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1375.811583] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-33b608b2-31c5-49aa-9bda-8581744c3914 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1375.819262] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97f00f21-c9c2-4e07-b4ff-c469046d4ca0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1375.847631] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7bd5cfba-6a92-4470-8c94-555d63d0a80c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1375.854088] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-26072aa4-fd76-4333-a6e6-ea7c1e375539 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1375.866777] env[67752]: DEBUG nova.compute.provider_tree [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1375.876799] env[67752]: DEBUG nova.scheduler.client.report [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1375.895224] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.385s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1375.895940] env[67752]: ERROR nova.compute.manager [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1375.895940] env[67752]: Faults: ['InvalidArgument'] [ 1375.895940] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Traceback (most recent call last): [ 1375.895940] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1375.895940] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] self.driver.spawn(context, instance, image_meta, [ 1375.895940] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1375.895940] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1375.895940] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1375.895940] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] self._fetch_image_if_missing(context, vi) [ 1375.895940] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1375.895940] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] image_cache(vi, tmp_image_ds_loc) [ 1375.895940] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1375.896365] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] vm_util.copy_virtual_disk( [ 1375.896365] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1375.896365] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] session._wait_for_task(vmdk_copy_task) [ 1375.896365] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1375.896365] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] return self.wait_for_task(task_ref) [ 1375.896365] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1375.896365] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] return evt.wait() [ 1375.896365] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1375.896365] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] result = hub.switch() [ 1375.896365] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1375.896365] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] return self.greenlet.switch() [ 1375.896365] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1375.896365] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] self.f(*self.args, **self.kw) [ 1375.896897] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1375.896897] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] raise exceptions.translate_fault(task_info.error) [ 1375.896897] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1375.896897] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Faults: ['InvalidArgument'] [ 1375.896897] env[67752]: ERROR nova.compute.manager [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] [ 1375.897083] env[67752]: DEBUG nova.compute.utils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1375.899215] env[67752]: DEBUG nova.compute.manager [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Build of instance ada70883-9bef-496e-a65f-ca84f6ee6211 was re-scheduled: A specified parameter was not correct: fileType [ 1375.899215] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1375.899215] env[67752]: DEBUG nova.compute.manager [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1375.899412] env[67752]: DEBUG nova.compute.manager [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1375.899624] env[67752]: DEBUG nova.compute.manager [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1375.899794] env[67752]: DEBUG nova.network.neutron [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1376.736876] env[67752]: DEBUG nova.network.neutron [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1376.751973] env[67752]: INFO nova.compute.manager [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Took 0.85 seconds to deallocate network for instance. [ 1376.855102] env[67752]: INFO nova.scheduler.client.report [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Deleted allocations for instance ada70883-9bef-496e-a65f-ca84f6ee6211 [ 1376.875992] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0031ae70-744a-4e21-9deb-f00b9a8fcdcc tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Lock "ada70883-9bef-496e-a65f-ca84f6ee6211" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 633.829s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1376.877066] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a0281c8f-05df-47c1-ae92-ab4685cca0aa tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Lock "ada70883-9bef-496e-a65f-ca84f6ee6211" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 437.303s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1376.877384] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a0281c8f-05df-47c1-ae92-ab4685cca0aa tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Acquiring lock "ada70883-9bef-496e-a65f-ca84f6ee6211-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1376.877530] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a0281c8f-05df-47c1-ae92-ab4685cca0aa tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Lock "ada70883-9bef-496e-a65f-ca84f6ee6211-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1376.877702] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a0281c8f-05df-47c1-ae92-ab4685cca0aa tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Lock "ada70883-9bef-496e-a65f-ca84f6ee6211-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1376.879634] env[67752]: INFO nova.compute.manager [None req-a0281c8f-05df-47c1-ae92-ab4685cca0aa tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Terminating instance [ 1376.881377] env[67752]: DEBUG nova.compute.manager [None req-a0281c8f-05df-47c1-ae92-ab4685cca0aa tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1376.881738] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a0281c8f-05df-47c1-ae92-ab4685cca0aa tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1376.882155] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-6046bae8-db9a-452e-be6f-4aca78539bce {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1376.887540] env[67752]: DEBUG nova.compute.manager [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1376.894309] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db04bfd0-a0e8-4fd1-94ab-ec71311baaa1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1376.923295] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-a0281c8f-05df-47c1-ae92-ab4685cca0aa tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance ada70883-9bef-496e-a65f-ca84f6ee6211 could not be found. [ 1376.923572] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-a0281c8f-05df-47c1-ae92-ab4685cca0aa tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1376.923757] env[67752]: INFO nova.compute.manager [None req-a0281c8f-05df-47c1-ae92-ab4685cca0aa tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1376.923992] env[67752]: DEBUG oslo.service.loopingcall [None req-a0281c8f-05df-47c1-ae92-ab4685cca0aa tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1376.928178] env[67752]: DEBUG nova.compute.manager [-] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1376.928282] env[67752]: DEBUG nova.network.neutron [-] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1376.940254] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1376.940487] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1376.941873] env[67752]: INFO nova.compute.claims [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1376.953056] env[67752]: DEBUG nova.network.neutron [-] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1376.970546] env[67752]: INFO nova.compute.manager [-] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] Took 0.04 seconds to deallocate network for instance. [ 1377.051903] env[67752]: DEBUG oslo_concurrency.lockutils [None req-a0281c8f-05df-47c1-ae92-ab4685cca0aa tempest-ServerMetadataTestJSON-1105300313 tempest-ServerMetadataTestJSON-1105300313-project-member] Lock "ada70883-9bef-496e-a65f-ca84f6ee6211" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.175s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1377.052826] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "ada70883-9bef-496e-a65f-ca84f6ee6211" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 197.412s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1377.053035] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: ada70883-9bef-496e-a65f-ca84f6ee6211] During sync_power_state the instance has a pending task (deleting). Skip. [ 1377.053219] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "ada70883-9bef-496e-a65f-ca84f6ee6211" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1377.176207] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fe5abaa0-f8d9-4499-a957-e2afcdc4e7cc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1377.183703] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9a5a8ed0-4688-453e-bf8d-04267ed7234d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1377.214052] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3855e7c3-ba9d-40fd-acea-40feeb1e1afb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1377.221262] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-575bfb6a-3034-4b39-b3c8-54ad689eb927 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1377.234616] env[67752]: DEBUG nova.compute.provider_tree [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1377.243865] env[67752]: DEBUG nova.scheduler.client.report [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1377.262505] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.322s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1377.262990] env[67752]: DEBUG nova.compute.manager [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1377.297294] env[67752]: DEBUG nova.compute.utils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1377.298902] env[67752]: DEBUG nova.compute.manager [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1377.298902] env[67752]: DEBUG nova.network.neutron [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1377.310864] env[67752]: DEBUG nova.compute.manager [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1377.371469] env[67752]: DEBUG nova.policy [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f31e3565acaa4672acaac19c5214b28e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e18ee59529f846859b4f8b8ed92852ed', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1377.378442] env[67752]: DEBUG nova.compute.manager [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1377.407463] env[67752]: DEBUG nova.virt.hardware [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1377.408645] env[67752]: DEBUG nova.virt.hardware [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1377.408645] env[67752]: DEBUG nova.virt.hardware [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1377.408645] env[67752]: DEBUG nova.virt.hardware [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1377.408645] env[67752]: DEBUG nova.virt.hardware [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1377.408645] env[67752]: DEBUG nova.virt.hardware [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1377.408934] env[67752]: DEBUG nova.virt.hardware [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1377.408934] env[67752]: DEBUG nova.virt.hardware [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1377.408934] env[67752]: DEBUG nova.virt.hardware [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1377.408934] env[67752]: DEBUG nova.virt.hardware [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1377.409213] env[67752]: DEBUG nova.virt.hardware [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1377.410084] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de045cca-5f97-46e1-a49e-feca2dcc54b9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1377.418197] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-156374f3-0fae-4221-bdb7-b5b01abd8393 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1377.760802] env[67752]: DEBUG nova.network.neutron [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Successfully created port: 34288f6c-d6f7-46dc-937c-653f5b8db9c2 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1378.239604] env[67752]: DEBUG nova.network.neutron [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Successfully created port: 617c953f-d39f-46f9-a0c9-94bf38772679 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1378.784681] env[67752]: DEBUG nova.network.neutron [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Successfully created port: 33118e51-a326-4f09-b0ac-3d244577263f {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1379.613311] env[67752]: DEBUG nova.compute.manager [req-b66e2269-0302-4965-ac6b-b46edf6e61fb req-eb899652-f56c-4780-895e-1991498dbcfe service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Received event network-vif-plugged-34288f6c-d6f7-46dc-937c-653f5b8db9c2 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1379.613535] env[67752]: DEBUG oslo_concurrency.lockutils [req-b66e2269-0302-4965-ac6b-b46edf6e61fb req-eb899652-f56c-4780-895e-1991498dbcfe service nova] Acquiring lock "81738fe6-6d50-4dfe-ae86-4c8771837168-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1379.613749] env[67752]: DEBUG oslo_concurrency.lockutils [req-b66e2269-0302-4965-ac6b-b46edf6e61fb req-eb899652-f56c-4780-895e-1991498dbcfe service nova] Lock "81738fe6-6d50-4dfe-ae86-4c8771837168-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1379.613921] env[67752]: DEBUG oslo_concurrency.lockutils [req-b66e2269-0302-4965-ac6b-b46edf6e61fb req-eb899652-f56c-4780-895e-1991498dbcfe service nova] Lock "81738fe6-6d50-4dfe-ae86-4c8771837168-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1379.614317] env[67752]: DEBUG nova.compute.manager [req-b66e2269-0302-4965-ac6b-b46edf6e61fb req-eb899652-f56c-4780-895e-1991498dbcfe service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] No waiting events found dispatching network-vif-plugged-34288f6c-d6f7-46dc-937c-653f5b8db9c2 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1379.614549] env[67752]: WARNING nova.compute.manager [req-b66e2269-0302-4965-ac6b-b46edf6e61fb req-eb899652-f56c-4780-895e-1991498dbcfe service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Received unexpected event network-vif-plugged-34288f6c-d6f7-46dc-937c-653f5b8db9c2 for instance with vm_state building and task_state spawning. [ 1379.630818] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1379.778439] env[67752]: DEBUG nova.network.neutron [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Successfully updated port: 34288f6c-d6f7-46dc-937c-653f5b8db9c2 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1380.912272] env[67752]: DEBUG nova.network.neutron [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Successfully updated port: 617c953f-d39f-46f9-a0c9-94bf38772679 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1381.637330] env[67752]: DEBUG nova.compute.manager [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Received event network-changed-34288f6c-d6f7-46dc-937c-653f5b8db9c2 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1381.637515] env[67752]: DEBUG nova.compute.manager [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Refreshing instance network info cache due to event network-changed-34288f6c-d6f7-46dc-937c-653f5b8db9c2. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1381.637705] env[67752]: DEBUG oslo_concurrency.lockutils [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] Acquiring lock "refresh_cache-81738fe6-6d50-4dfe-ae86-4c8771837168" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1381.637854] env[67752]: DEBUG oslo_concurrency.lockutils [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] Acquired lock "refresh_cache-81738fe6-6d50-4dfe-ae86-4c8771837168" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1381.638035] env[67752]: DEBUG nova.network.neutron [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Refreshing network info cache for port 34288f6c-d6f7-46dc-937c-653f5b8db9c2 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1381.709408] env[67752]: DEBUG nova.network.neutron [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1381.920657] env[67752]: DEBUG nova.network.neutron [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Successfully updated port: 33118e51-a326-4f09-b0ac-3d244577263f {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1381.928727] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquiring lock "refresh_cache-81738fe6-6d50-4dfe-ae86-4c8771837168" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1381.959909] env[67752]: DEBUG nova.network.neutron [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1381.968939] env[67752]: DEBUG oslo_concurrency.lockutils [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] Releasing lock "refresh_cache-81738fe6-6d50-4dfe-ae86-4c8771837168" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1381.969279] env[67752]: DEBUG nova.compute.manager [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Received event network-vif-plugged-617c953f-d39f-46f9-a0c9-94bf38772679 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1381.969483] env[67752]: DEBUG oslo_concurrency.lockutils [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] Acquiring lock "81738fe6-6d50-4dfe-ae86-4c8771837168-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1381.969695] env[67752]: DEBUG oslo_concurrency.lockutils [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] Lock "81738fe6-6d50-4dfe-ae86-4c8771837168-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1381.969858] env[67752]: DEBUG oslo_concurrency.lockutils [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] Lock "81738fe6-6d50-4dfe-ae86-4c8771837168-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1381.970034] env[67752]: DEBUG nova.compute.manager [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] No waiting events found dispatching network-vif-plugged-617c953f-d39f-46f9-a0c9-94bf38772679 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1381.970211] env[67752]: WARNING nova.compute.manager [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Received unexpected event network-vif-plugged-617c953f-d39f-46f9-a0c9-94bf38772679 for instance with vm_state building and task_state spawning. [ 1381.970374] env[67752]: DEBUG nova.compute.manager [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Received event network-changed-617c953f-d39f-46f9-a0c9-94bf38772679 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1381.970544] env[67752]: DEBUG nova.compute.manager [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Refreshing instance network info cache due to event network-changed-617c953f-d39f-46f9-a0c9-94bf38772679. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1381.970751] env[67752]: DEBUG oslo_concurrency.lockutils [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] Acquiring lock "refresh_cache-81738fe6-6d50-4dfe-ae86-4c8771837168" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1381.970894] env[67752]: DEBUG oslo_concurrency.lockutils [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] Acquired lock "refresh_cache-81738fe6-6d50-4dfe-ae86-4c8771837168" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1381.971065] env[67752]: DEBUG nova.network.neutron [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Refreshing network info cache for port 617c953f-d39f-46f9-a0c9-94bf38772679 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1382.015165] env[67752]: DEBUG nova.network.neutron [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1382.202873] env[67752]: DEBUG nova.network.neutron [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1382.212778] env[67752]: DEBUG oslo_concurrency.lockutils [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] Releasing lock "refresh_cache-81738fe6-6d50-4dfe-ae86-4c8771837168" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1382.213795] env[67752]: DEBUG nova.compute.manager [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Received event network-vif-plugged-33118e51-a326-4f09-b0ac-3d244577263f {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1382.213795] env[67752]: DEBUG oslo_concurrency.lockutils [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] Acquiring lock "81738fe6-6d50-4dfe-ae86-4c8771837168-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1382.213795] env[67752]: DEBUG oslo_concurrency.lockutils [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] Lock "81738fe6-6d50-4dfe-ae86-4c8771837168-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1382.213795] env[67752]: DEBUG oslo_concurrency.lockutils [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] Lock "81738fe6-6d50-4dfe-ae86-4c8771837168-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1382.213962] env[67752]: DEBUG nova.compute.manager [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] No waiting events found dispatching network-vif-plugged-33118e51-a326-4f09-b0ac-3d244577263f {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1382.213962] env[67752]: WARNING nova.compute.manager [req-de960d4c-c188-40bb-952e-f67be0753393 req-d21fa322-223d-40e3-aa77-0888a6a271dc service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Received unexpected event network-vif-plugged-33118e51-a326-4f09-b0ac-3d244577263f for instance with vm_state building and task_state spawning. [ 1382.214242] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquired lock "refresh_cache-81738fe6-6d50-4dfe-ae86-4c8771837168" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1382.214368] env[67752]: DEBUG nova.network.neutron [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1382.267753] env[67752]: DEBUG nova.network.neutron [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1382.630447] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1382.663819] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1382.663989] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 1382.664125] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 1382.692389] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1382.692562] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1382.692693] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1382.692818] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1382.692939] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: d3940c29-852a-427b-9027-aa4080150724] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1382.693073] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1382.693196] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1382.693316] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1382.693436] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1382.693555] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1382.693675] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 1382.694128] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1383.151151] env[67752]: DEBUG nova.network.neutron [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Updating instance_info_cache with network_info: [{"id": "34288f6c-d6f7-46dc-937c-653f5b8db9c2", "address": "fa:16:3e:2f:68:0d", "network": {"id": "84d584bd-07f7-4d0f-a667-93c2f21f269b", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-712230662", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.66", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e18ee59529f846859b4f8b8ed92852ed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "674802e7-b847-4bef-a7a8-f90ac7a3a0a7", "external-id": "nsx-vlan-transportzone-953", "segmentation_id": 953, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap34288f6c-d6", "ovs_interfaceid": "34288f6c-d6f7-46dc-937c-653f5b8db9c2", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "617c953f-d39f-46f9-a0c9-94bf38772679", "address": "fa:16:3e:13:58:4f", "network": {"id": "9cac9bda-9ce5-424c-a869-e9b5a6f00f2e", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-333526587", "subnets": [{"cidr": "192.168.129.0/24", "dns": [], "gateway": {"address": "192.168.129.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.129.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.129.2"}}], "meta": {"injected": false, "tenant_id": "e18ee59529f846859b4f8b8ed92852ed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "33ddef78-922c-4cd3-99b0-971ac7802856", "external-id": "nsx-vlan-transportzone-311", "segmentation_id": 311, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap617c953f-d3", "ovs_interfaceid": "617c953f-d39f-46f9-a0c9-94bf38772679", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "33118e51-a326-4f09-b0ac-3d244577263f", "address": "fa:16:3e:18:8e:b6", "network": {"id": "84d584bd-07f7-4d0f-a667-93c2f21f269b", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-712230662", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.87", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e18ee59529f846859b4f8b8ed92852ed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "674802e7-b847-4bef-a7a8-f90ac7a3a0a7", "external-id": "nsx-vlan-transportzone-953", "segmentation_id": 953, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap33118e51-a3", "ovs_interfaceid": "33118e51-a326-4f09-b0ac-3d244577263f", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1383.167903] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Releasing lock "refresh_cache-81738fe6-6d50-4dfe-ae86-4c8771837168" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1383.168291] env[67752]: DEBUG nova.compute.manager [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Instance network_info: |[{"id": "34288f6c-d6f7-46dc-937c-653f5b8db9c2", "address": "fa:16:3e:2f:68:0d", "network": {"id": "84d584bd-07f7-4d0f-a667-93c2f21f269b", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-712230662", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.66", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e18ee59529f846859b4f8b8ed92852ed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "674802e7-b847-4bef-a7a8-f90ac7a3a0a7", "external-id": "nsx-vlan-transportzone-953", "segmentation_id": 953, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap34288f6c-d6", "ovs_interfaceid": "34288f6c-d6f7-46dc-937c-653f5b8db9c2", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "617c953f-d39f-46f9-a0c9-94bf38772679", "address": "fa:16:3e:13:58:4f", "network": {"id": "9cac9bda-9ce5-424c-a869-e9b5a6f00f2e", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-333526587", "subnets": [{"cidr": "192.168.129.0/24", "dns": [], "gateway": {"address": "192.168.129.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.129.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.129.2"}}], "meta": {"injected": false, "tenant_id": "e18ee59529f846859b4f8b8ed92852ed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "33ddef78-922c-4cd3-99b0-971ac7802856", "external-id": "nsx-vlan-transportzone-311", "segmentation_id": 311, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap617c953f-d3", "ovs_interfaceid": "617c953f-d39f-46f9-a0c9-94bf38772679", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "33118e51-a326-4f09-b0ac-3d244577263f", "address": "fa:16:3e:18:8e:b6", "network": {"id": "84d584bd-07f7-4d0f-a667-93c2f21f269b", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-712230662", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.87", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e18ee59529f846859b4f8b8ed92852ed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "674802e7-b847-4bef-a7a8-f90ac7a3a0a7", "external-id": "nsx-vlan-transportzone-953", "segmentation_id": 953, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap33118e51-a3", "ovs_interfaceid": "33118e51-a326-4f09-b0ac-3d244577263f", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1383.168752] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:2f:68:0d', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '674802e7-b847-4bef-a7a8-f90ac7a3a0a7', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '34288f6c-d6f7-46dc-937c-653f5b8db9c2', 'vif_model': 'vmxnet3'}, {'network_name': 'br-int', 'mac_address': 'fa:16:3e:13:58:4f', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '33ddef78-922c-4cd3-99b0-971ac7802856', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '617c953f-d39f-46f9-a0c9-94bf38772679', 'vif_model': 'vmxnet3'}, {'network_name': 'br-int', 'mac_address': 'fa:16:3e:18:8e:b6', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '674802e7-b847-4bef-a7a8-f90ac7a3a0a7', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '33118e51-a326-4f09-b0ac-3d244577263f', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1383.179669] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Creating folder: Project (e18ee59529f846859b4f8b8ed92852ed). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1383.180195] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-6e5ca365-fdb0-4aad-b834-8875cc9efa5d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1383.191982] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Created folder: Project (e18ee59529f846859b4f8b8ed92852ed) in parent group-v639722. [ 1383.192182] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Creating folder: Instances. Parent ref: group-v639812. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1383.192404] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-fc142628-5fb8-41b1-983c-b9fe15c9b7d6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1383.200835] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Created folder: Instances in parent group-v639812. [ 1383.201073] env[67752]: DEBUG oslo.service.loopingcall [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1383.201285] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1383.201467] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-0ecb623e-78c1-484f-bb13-a3dd5032c14e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1383.224386] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1383.224386] env[67752]: value = "task-3199775" [ 1383.224386] env[67752]: _type = "Task" [ 1383.224386] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1383.232949] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199775, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1383.635239] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1383.671921] env[67752]: DEBUG nova.compute.manager [req-fa6a15e3-8fa9-4680-bba2-eead0394ff19 req-1609ad21-f3b3-4712-bd37-fea6031dbfc2 service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Received event network-changed-33118e51-a326-4f09-b0ac-3d244577263f {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1383.672144] env[67752]: DEBUG nova.compute.manager [req-fa6a15e3-8fa9-4680-bba2-eead0394ff19 req-1609ad21-f3b3-4712-bd37-fea6031dbfc2 service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Refreshing instance network info cache due to event network-changed-33118e51-a326-4f09-b0ac-3d244577263f. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1383.672397] env[67752]: DEBUG oslo_concurrency.lockutils [req-fa6a15e3-8fa9-4680-bba2-eead0394ff19 req-1609ad21-f3b3-4712-bd37-fea6031dbfc2 service nova] Acquiring lock "refresh_cache-81738fe6-6d50-4dfe-ae86-4c8771837168" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1383.672501] env[67752]: DEBUG oslo_concurrency.lockutils [req-fa6a15e3-8fa9-4680-bba2-eead0394ff19 req-1609ad21-f3b3-4712-bd37-fea6031dbfc2 service nova] Acquired lock "refresh_cache-81738fe6-6d50-4dfe-ae86-4c8771837168" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1383.672678] env[67752]: DEBUG nova.network.neutron [req-fa6a15e3-8fa9-4680-bba2-eead0394ff19 req-1609ad21-f3b3-4712-bd37-fea6031dbfc2 service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Refreshing network info cache for port 33118e51-a326-4f09-b0ac-3d244577263f {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1383.735702] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199775, 'name': CreateVM_Task, 'duration_secs': 0.369539} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1383.735702] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1383.736594] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1383.736765] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1383.737163] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1383.737422] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-44a9ec4d-8bfa-4742-85f9-b0b5b5c1c529 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1383.743074] env[67752]: DEBUG oslo_vmware.api [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Waiting for the task: (returnval){ [ 1383.743074] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]520a7c64-697f-edd7-a9bd-4ea4bb2daf1b" [ 1383.743074] env[67752]: _type = "Task" [ 1383.743074] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1383.751920] env[67752]: DEBUG oslo_vmware.api [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]520a7c64-697f-edd7-a9bd-4ea4bb2daf1b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1384.061507] env[67752]: DEBUG nova.network.neutron [req-fa6a15e3-8fa9-4680-bba2-eead0394ff19 req-1609ad21-f3b3-4712-bd37-fea6031dbfc2 service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Updated VIF entry in instance network info cache for port 33118e51-a326-4f09-b0ac-3d244577263f. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1384.061983] env[67752]: DEBUG nova.network.neutron [req-fa6a15e3-8fa9-4680-bba2-eead0394ff19 req-1609ad21-f3b3-4712-bd37-fea6031dbfc2 service nova] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Updating instance_info_cache with network_info: [{"id": "34288f6c-d6f7-46dc-937c-653f5b8db9c2", "address": "fa:16:3e:2f:68:0d", "network": {"id": "84d584bd-07f7-4d0f-a667-93c2f21f269b", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-712230662", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.66", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e18ee59529f846859b4f8b8ed92852ed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "674802e7-b847-4bef-a7a8-f90ac7a3a0a7", "external-id": "nsx-vlan-transportzone-953", "segmentation_id": 953, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap34288f6c-d6", "ovs_interfaceid": "34288f6c-d6f7-46dc-937c-653f5b8db9c2", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "617c953f-d39f-46f9-a0c9-94bf38772679", "address": "fa:16:3e:13:58:4f", "network": {"id": "9cac9bda-9ce5-424c-a869-e9b5a6f00f2e", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-333526587", "subnets": [{"cidr": "192.168.129.0/24", "dns": [], "gateway": {"address": "192.168.129.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.129.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.129.2"}}], "meta": {"injected": false, "tenant_id": "e18ee59529f846859b4f8b8ed92852ed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "33ddef78-922c-4cd3-99b0-971ac7802856", "external-id": "nsx-vlan-transportzone-311", "segmentation_id": 311, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap617c953f-d3", "ovs_interfaceid": "617c953f-d39f-46f9-a0c9-94bf38772679", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "33118e51-a326-4f09-b0ac-3d244577263f", "address": "fa:16:3e:18:8e:b6", "network": {"id": "84d584bd-07f7-4d0f-a667-93c2f21f269b", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-712230662", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.87", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e18ee59529f846859b4f8b8ed92852ed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "674802e7-b847-4bef-a7a8-f90ac7a3a0a7", "external-id": "nsx-vlan-transportzone-953", "segmentation_id": 953, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap33118e51-a3", "ovs_interfaceid": "33118e51-a326-4f09-b0ac-3d244577263f", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1384.072305] env[67752]: DEBUG oslo_concurrency.lockutils [req-fa6a15e3-8fa9-4680-bba2-eead0394ff19 req-1609ad21-f3b3-4712-bd37-fea6031dbfc2 service nova] Releasing lock "refresh_cache-81738fe6-6d50-4dfe-ae86-4c8771837168" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1384.253656] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1384.253929] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1384.254141] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1384.635117] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1384.635357] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1386.635391] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1386.635683] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 1389.634822] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1389.634822] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1389.646496] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1389.646716] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1389.646889] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1389.647069] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1389.648290] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-18f4de1b-05d2-454f-8e8b-165c6840a4eb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1389.657258] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-323335bd-406a-4202-b71c-9cb9c40ad608 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1389.670699] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-33066b47-0591-48ad-8831-a07bdd5d9b7b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1389.677564] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f695400-03c6-4651-8ead-a52b07557411 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1389.707738] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181006MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1389.707912] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1389.708113] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1389.804132] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1389.804370] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b271cd73-a148-48d6-bed9-6a99512457bd actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1389.804550] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1389.804680] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f6499ee5-cb24-44e1-9fe7-c19feeb732eb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1389.804803] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d3940c29-852a-427b-9027-aa4080150724 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1389.804923] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1389.805053] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance a8bd310e-530a-46ab-add7-1b827ea5f399 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1389.805173] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9432cd04-c2df-428b-b31a-93d3ec0c9d1f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1389.805288] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 90e58033-eacd-433a-8d25-40f7d8403588 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1389.805410] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 81738fe6-6d50-4dfe-ae86-4c8771837168 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1389.816980] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e4ebb0f0-ab58-491f-bc4a-7548bbe415d8 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1389.828443] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance ba83b143-d752-4166-8ce0-fbd7fd4875a4 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1389.838338] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 6000d3ce-6bdc-4683-a1a9-fbc1e447aad4 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1389.847898] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fe397d83-8625-4c5c-88b2-6c96c51c42bf has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1389.857283] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9ce8f114-c06d-4f08-b64a-980b94975ff7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1389.866714] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 5142518d-3b2a-474c-81d9-d658cb8467e3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1389.875675] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c1975c17-15a5-4751-bf7f-d5b7e0d2fea1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1389.875899] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1389.876057] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1390.085950] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-05204ee7-7809-4aba-8d8e-854e080550a7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1390.093157] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b26f5ea-d613-485b-b48e-34353b6dfbee {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1390.122206] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dfde473b-0a27-4331-9120-fbcf5fa781ad {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1390.129620] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-595097c7-a9b9-490e-88d7-420036ecca65 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1390.143486] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1390.151772] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1390.166278] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1390.166472] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.458s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1411.077121] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "0f3f8622-4b01-47d3-bbbb-dcd88c7ca010" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1411.077421] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "0f3f8622-4b01-47d3-bbbb-dcd88c7ca010" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1411.107512] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "7557e3a1-b298-401f-8320-5a48ddda3298" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1411.107764] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "7557e3a1-b298-401f-8320-5a48ddda3298" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1417.244952] env[67752]: DEBUG oslo_concurrency.lockutils [None req-747b37fc-be61-4aba-aa0f-8c0cde40f237 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Acquiring lock "90e58033-eacd-433a-8d25-40f7d8403588" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1421.752490] env[67752]: DEBUG oslo_concurrency.lockutils [None req-70799d8f-a860-4c0e-b81a-d37df565547d tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquiring lock "81738fe6-6d50-4dfe-ae86-4c8771837168" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1423.308109] env[67752]: WARNING oslo_vmware.rw_handles [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1423.308109] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1423.308109] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1423.308109] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1423.308109] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1423.308109] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1423.308109] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1423.308109] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1423.308109] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1423.308109] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1423.308109] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1423.308109] env[67752]: ERROR oslo_vmware.rw_handles [ 1423.308109] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/05471069-e7c2-401f-8c70-9b70fbcc167b/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1423.310491] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1423.310749] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Copying Virtual Disk [datastore2] vmware_temp/05471069-e7c2-401f-8c70-9b70fbcc167b/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/05471069-e7c2-401f-8c70-9b70fbcc167b/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1423.311048] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-a1e3bd42-16d1-407a-a97d-7b6eb5e40d0f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1423.318637] env[67752]: DEBUG oslo_vmware.api [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Waiting for the task: (returnval){ [ 1423.318637] env[67752]: value = "task-3199776" [ 1423.318637] env[67752]: _type = "Task" [ 1423.318637] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1423.328264] env[67752]: DEBUG oslo_vmware.api [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Task: {'id': task-3199776, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1423.830070] env[67752]: DEBUG oslo_vmware.exceptions [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1423.830070] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1423.830070] env[67752]: ERROR nova.compute.manager [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1423.830070] env[67752]: Faults: ['InvalidArgument'] [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Traceback (most recent call last): [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] yield resources [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] self.driver.spawn(context, instance, image_meta, [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] self._fetch_image_if_missing(context, vi) [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] image_cache(vi, tmp_image_ds_loc) [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] vm_util.copy_virtual_disk( [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] session._wait_for_task(vmdk_copy_task) [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] return self.wait_for_task(task_ref) [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] return evt.wait() [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] result = hub.switch() [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] return self.greenlet.switch() [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] self.f(*self.args, **self.kw) [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] raise exceptions.translate_fault(task_info.error) [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Faults: ['InvalidArgument'] [ 1423.830070] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] [ 1423.830070] env[67752]: INFO nova.compute.manager [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Terminating instance [ 1423.831438] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1423.831644] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1423.831894] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-bb91bf61-4c69-475b-8580-fe9ab8082cc5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1423.834918] env[67752]: DEBUG nova.compute.manager [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1423.835200] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1423.835937] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9a0b5d69-0ae9-4426-b8d0-72564920c7e8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1423.842992] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1423.843293] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-56d3b0a6-437e-4589-a5f4-d9c7df6bdc97 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1423.845578] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1423.845790] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1423.846782] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-0de8ee65-5a99-4c33-877c-06d38a55e517 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1423.851496] env[67752]: DEBUG oslo_vmware.api [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Waiting for the task: (returnval){ [ 1423.851496] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5221ce60-2778-65b4-d60f-eec241d5a80a" [ 1423.851496] env[67752]: _type = "Task" [ 1423.851496] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1423.865776] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1423.866025] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Creating directory with path [datastore2] vmware_temp/bea47f13-4429-472c-943b-0b3645c0e181/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1423.866262] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-caaf279d-348f-46fe-b6ca-1359251651e7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1423.886706] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Created directory with path [datastore2] vmware_temp/bea47f13-4429-472c-943b-0b3645c0e181/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1423.886892] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Fetch image to [datastore2] vmware_temp/bea47f13-4429-472c-943b-0b3645c0e181/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1423.887080] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/bea47f13-4429-472c-943b-0b3645c0e181/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1423.887805] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c377135-14c7-4c5f-942f-3a77c3328a74 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1423.894662] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a65077e-33e5-4efb-8da0-3d97818c5495 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1423.903856] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50d4486f-8591-450b-aad8-f9daf498cf2e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1423.934560] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b337a6ec-9495-4862-92e7-836d6eeeedd9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1423.936901] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1423.937112] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1423.937290] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Deleting the datastore file [datastore2] 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1423.937497] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-41f754d9-6f38-4e4f-b145-df07c18e7941 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1423.943905] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-96cc9332-c72d-4d25-8edb-f99598b7d091 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1423.946131] env[67752]: DEBUG oslo_vmware.api [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Waiting for the task: (returnval){ [ 1423.946131] env[67752]: value = "task-3199778" [ 1423.946131] env[67752]: _type = "Task" [ 1423.946131] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1423.955130] env[67752]: DEBUG oslo_vmware.api [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Task: {'id': task-3199778, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1423.968841] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1424.019129] env[67752]: DEBUG oslo_vmware.rw_handles [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/bea47f13-4429-472c-943b-0b3645c0e181/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1424.080976] env[67752]: DEBUG oslo_vmware.rw_handles [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1424.080976] env[67752]: DEBUG oslo_vmware.rw_handles [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Closing write handle for https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/bea47f13-4429-472c-943b-0b3645c0e181/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1424.457787] env[67752]: DEBUG oslo_vmware.api [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Task: {'id': task-3199778, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.065231} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1424.458177] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1424.458220] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1424.459030] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1424.459030] env[67752]: INFO nova.compute.manager [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Took 0.62 seconds to destroy the instance on the hypervisor. [ 1424.460698] env[67752]: DEBUG nova.compute.claims [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1424.460886] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1424.461113] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1424.724921] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f3e2512c-88d6-4e4f-9da4-5af408ff2775 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1424.732999] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-23e8f783-6888-4599-b034-c7d62d6ff996 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1424.762441] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5578847f-0712-4b4c-8476-fa8158ac59dc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1424.769470] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a9047576-6ce0-4e29-80f1-107cb0469402 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1424.782466] env[67752]: DEBUG nova.compute.provider_tree [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1424.791355] env[67752]: DEBUG nova.scheduler.client.report [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1424.806243] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.345s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1424.806755] env[67752]: ERROR nova.compute.manager [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1424.806755] env[67752]: Faults: ['InvalidArgument'] [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Traceback (most recent call last): [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] self.driver.spawn(context, instance, image_meta, [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] self._fetch_image_if_missing(context, vi) [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] image_cache(vi, tmp_image_ds_loc) [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] vm_util.copy_virtual_disk( [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] session._wait_for_task(vmdk_copy_task) [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] return self.wait_for_task(task_ref) [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] return evt.wait() [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] result = hub.switch() [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] return self.greenlet.switch() [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] self.f(*self.args, **self.kw) [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] raise exceptions.translate_fault(task_info.error) [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Faults: ['InvalidArgument'] [ 1424.806755] env[67752]: ERROR nova.compute.manager [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] [ 1424.807565] env[67752]: DEBUG nova.compute.utils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1424.811138] env[67752]: DEBUG nova.compute.manager [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Build of instance 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70 was re-scheduled: A specified parameter was not correct: fileType [ 1424.811138] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1424.811511] env[67752]: DEBUG nova.compute.manager [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1424.811685] env[67752]: DEBUG nova.compute.manager [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1424.811866] env[67752]: DEBUG nova.compute.manager [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1424.812067] env[67752]: DEBUG nova.network.neutron [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1425.200028] env[67752]: DEBUG nova.network.neutron [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1425.215194] env[67752]: INFO nova.compute.manager [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Took 0.40 seconds to deallocate network for instance. [ 1425.304484] env[67752]: INFO nova.scheduler.client.report [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Deleted allocations for instance 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70 [ 1425.330034] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2827af27-5ed1-4491-80fd-e99ea83d9c90 tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Lock "1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 628.392s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1425.330617] env[67752]: DEBUG oslo_concurrency.lockutils [None req-41977d13-8ea4-4f2f-ab4f-655b474eb19b tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Lock "1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 431.815s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1425.330916] env[67752]: DEBUG oslo_concurrency.lockutils [None req-41977d13-8ea4-4f2f-ab4f-655b474eb19b tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Acquiring lock "1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1425.331185] env[67752]: DEBUG oslo_concurrency.lockutils [None req-41977d13-8ea4-4f2f-ab4f-655b474eb19b tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Lock "1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1425.331283] env[67752]: DEBUG oslo_concurrency.lockutils [None req-41977d13-8ea4-4f2f-ab4f-655b474eb19b tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Lock "1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1425.333320] env[67752]: INFO nova.compute.manager [None req-41977d13-8ea4-4f2f-ab4f-655b474eb19b tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Terminating instance [ 1425.335014] env[67752]: DEBUG nova.compute.manager [None req-41977d13-8ea4-4f2f-ab4f-655b474eb19b tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1425.336024] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-41977d13-8ea4-4f2f-ab4f-655b474eb19b tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1425.336024] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-db933d08-b1b9-477d-9582-c713485868a6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1425.344697] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42faa4e9-dd18-4aaa-be06-59be56c6c5cc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1425.356541] env[67752]: DEBUG nova.compute.manager [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1425.376788] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-41977d13-8ea4-4f2f-ab4f-655b474eb19b tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70 could not be found. [ 1425.377009] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-41977d13-8ea4-4f2f-ab4f-655b474eb19b tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1425.381013] env[67752]: INFO nova.compute.manager [None req-41977d13-8ea4-4f2f-ab4f-655b474eb19b tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1425.381013] env[67752]: DEBUG oslo.service.loopingcall [None req-41977d13-8ea4-4f2f-ab4f-655b474eb19b tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1425.381013] env[67752]: DEBUG nova.compute.manager [-] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1425.381013] env[67752]: DEBUG nova.network.neutron [-] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1425.409635] env[67752]: DEBUG nova.network.neutron [-] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1425.423237] env[67752]: INFO nova.compute.manager [-] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] Took 0.05 seconds to deallocate network for instance. [ 1425.432090] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1425.432090] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1425.433330] env[67752]: INFO nova.compute.claims [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1425.525607] env[67752]: DEBUG oslo_concurrency.lockutils [None req-41977d13-8ea4-4f2f-ab4f-655b474eb19b tempest-ServerAddressesTestJSON-724630278 tempest-ServerAddressesTestJSON-724630278-project-member] Lock "1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.195s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1425.526629] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 245.886s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1425.526822] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70] During sync_power_state the instance has a pending task (deleting). Skip. [ 1425.527029] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "1c096b5f-fdb0-4ad1-a5a8-2c1ae0e88e70" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1425.707588] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0fd854a-38ae-4800-adcf-2a389c746f7a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1425.715268] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-12fbfae8-5ec8-4739-ae5a-d65a6aaa06ab {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1425.745522] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-20746429-9595-45a9-9df6-d0be3172d4fe {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1425.752533] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-be400308-00aa-49e0-9944-2fee5683e888 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1425.765332] env[67752]: DEBUG nova.compute.provider_tree [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1425.773492] env[67752]: DEBUG nova.scheduler.client.report [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1425.791029] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.359s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1425.791029] env[67752]: DEBUG nova.compute.manager [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1425.822048] env[67752]: DEBUG nova.compute.utils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1425.823540] env[67752]: DEBUG nova.compute.manager [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1425.823722] env[67752]: DEBUG nova.network.neutron [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1425.831689] env[67752]: DEBUG nova.compute.manager [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1425.885012] env[67752]: DEBUG nova.policy [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '3e9e79d9355e49e1a793cc10423de7b1', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd1cb577ba011491ba686908eea96d08f', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1425.894354] env[67752]: DEBUG nova.compute.manager [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1425.919104] env[67752]: DEBUG nova.virt.hardware [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1425.919366] env[67752]: DEBUG nova.virt.hardware [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1425.919527] env[67752]: DEBUG nova.virt.hardware [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1425.919711] env[67752]: DEBUG nova.virt.hardware [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1425.919864] env[67752]: DEBUG nova.virt.hardware [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1425.920016] env[67752]: DEBUG nova.virt.hardware [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1425.920238] env[67752]: DEBUG nova.virt.hardware [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1425.920413] env[67752]: DEBUG nova.virt.hardware [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1425.920583] env[67752]: DEBUG nova.virt.hardware [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1425.920761] env[67752]: DEBUG nova.virt.hardware [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1425.920936] env[67752]: DEBUG nova.virt.hardware [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1425.921797] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee67a2c2-c97b-4a4f-a982-d66f38c1a1b0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1425.929751] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-feeea012-44d2-4a78-8852-40bae0db04cb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1426.209124] env[67752]: DEBUG nova.network.neutron [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Successfully created port: 19116fde-f7a5-47c4-9e70-5958691ec5be {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1427.248161] env[67752]: DEBUG nova.network.neutron [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Successfully updated port: 19116fde-f7a5-47c4-9e70-5958691ec5be {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1427.259030] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Acquiring lock "refresh_cache-e4ebb0f0-ab58-491f-bc4a-7548bbe415d8" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1427.259221] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Acquired lock "refresh_cache-e4ebb0f0-ab58-491f-bc4a-7548bbe415d8" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1427.259381] env[67752]: DEBUG nova.network.neutron [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1427.353304] env[67752]: DEBUG nova.network.neutron [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1427.544312] env[67752]: DEBUG nova.compute.manager [req-d143f6b8-fc23-44a6-8260-225ec7dbce2e req-fc0ef099-b0d0-4a10-9ba4-d930bd9e8d13 service nova] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Received event network-vif-plugged-19116fde-f7a5-47c4-9e70-5958691ec5be {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1427.544553] env[67752]: DEBUG oslo_concurrency.lockutils [req-d143f6b8-fc23-44a6-8260-225ec7dbce2e req-fc0ef099-b0d0-4a10-9ba4-d930bd9e8d13 service nova] Acquiring lock "e4ebb0f0-ab58-491f-bc4a-7548bbe415d8-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1427.544854] env[67752]: DEBUG oslo_concurrency.lockutils [req-d143f6b8-fc23-44a6-8260-225ec7dbce2e req-fc0ef099-b0d0-4a10-9ba4-d930bd9e8d13 service nova] Lock "e4ebb0f0-ab58-491f-bc4a-7548bbe415d8-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1427.544995] env[67752]: DEBUG oslo_concurrency.lockutils [req-d143f6b8-fc23-44a6-8260-225ec7dbce2e req-fc0ef099-b0d0-4a10-9ba4-d930bd9e8d13 service nova] Lock "e4ebb0f0-ab58-491f-bc4a-7548bbe415d8-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1427.545368] env[67752]: DEBUG nova.compute.manager [req-d143f6b8-fc23-44a6-8260-225ec7dbce2e req-fc0ef099-b0d0-4a10-9ba4-d930bd9e8d13 service nova] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] No waiting events found dispatching network-vif-plugged-19116fde-f7a5-47c4-9e70-5958691ec5be {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1427.545587] env[67752]: WARNING nova.compute.manager [req-d143f6b8-fc23-44a6-8260-225ec7dbce2e req-fc0ef099-b0d0-4a10-9ba4-d930bd9e8d13 service nova] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Received unexpected event network-vif-plugged-19116fde-f7a5-47c4-9e70-5958691ec5be for instance with vm_state building and task_state spawning. [ 1427.614280] env[67752]: DEBUG nova.network.neutron [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Updating instance_info_cache with network_info: [{"id": "19116fde-f7a5-47c4-9e70-5958691ec5be", "address": "fa:16:3e:c7:a3:a4", "network": {"id": "9c1474f0-0949-46b4-a9a2-e00010321d55", "bridge": "br-int", "label": "tempest-ServersNegativeTestMultiTenantJSON-153947399-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d1cb577ba011491ba686908eea96d08f", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "8f40f5c4-c146-449c-884d-6f884dcf2acf", "external-id": "nsx-vlan-transportzone-240", "segmentation_id": 240, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap19116fde-f7", "ovs_interfaceid": "19116fde-f7a5-47c4-9e70-5958691ec5be", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1427.625649] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Releasing lock "refresh_cache-e4ebb0f0-ab58-491f-bc4a-7548bbe415d8" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1427.625944] env[67752]: DEBUG nova.compute.manager [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Instance network_info: |[{"id": "19116fde-f7a5-47c4-9e70-5958691ec5be", "address": "fa:16:3e:c7:a3:a4", "network": {"id": "9c1474f0-0949-46b4-a9a2-e00010321d55", "bridge": "br-int", "label": "tempest-ServersNegativeTestMultiTenantJSON-153947399-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d1cb577ba011491ba686908eea96d08f", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "8f40f5c4-c146-449c-884d-6f884dcf2acf", "external-id": "nsx-vlan-transportzone-240", "segmentation_id": 240, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap19116fde-f7", "ovs_interfaceid": "19116fde-f7a5-47c4-9e70-5958691ec5be", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1427.626369] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:c7:a3:a4', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '8f40f5c4-c146-449c-884d-6f884dcf2acf', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '19116fde-f7a5-47c4-9e70-5958691ec5be', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1427.634441] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Creating folder: Project (d1cb577ba011491ba686908eea96d08f). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1427.634708] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-08f2171b-f772-4d45-9dde-bf85628e03fb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1427.645110] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Created folder: Project (d1cb577ba011491ba686908eea96d08f) in parent group-v639722. [ 1427.645354] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Creating folder: Instances. Parent ref: group-v639815. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1427.645610] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-564963f3-5ba4-4c0f-9326-74c3d215da44 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1427.654192] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Created folder: Instances in parent group-v639815. [ 1427.654485] env[67752]: DEBUG oslo.service.loopingcall [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1427.654700] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1427.654918] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-3f644b04-33ea-4e29-9245-bc2f6a22b71a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1427.675216] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1427.675216] env[67752]: value = "task-3199781" [ 1427.675216] env[67752]: _type = "Task" [ 1427.675216] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1427.682469] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199781, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1428.185613] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199781, 'name': CreateVM_Task, 'duration_secs': 0.273982} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1428.185781] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1428.186472] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1428.186642] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1428.186968] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1428.187244] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-83e308fc-1053-4bea-a9cc-673dbfbb6aaa {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1428.191831] env[67752]: DEBUG oslo_vmware.api [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Waiting for the task: (returnval){ [ 1428.191831] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]525f8384-c232-d37e-5ad1-d1a66d69b64e" [ 1428.191831] env[67752]: _type = "Task" [ 1428.191831] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1428.199524] env[67752]: DEBUG oslo_vmware.api [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]525f8384-c232-d37e-5ad1-d1a66d69b64e, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1428.701606] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1428.701884] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1428.702243] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1429.570110] env[67752]: DEBUG nova.compute.manager [req-d0e3a657-7b89-4f6c-bcaa-a8d1eebef245 req-943b8b7f-de0d-4c03-86db-fc59412ea760 service nova] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Received event network-changed-19116fde-f7a5-47c4-9e70-5958691ec5be {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1429.570342] env[67752]: DEBUG nova.compute.manager [req-d0e3a657-7b89-4f6c-bcaa-a8d1eebef245 req-943b8b7f-de0d-4c03-86db-fc59412ea760 service nova] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Refreshing instance network info cache due to event network-changed-19116fde-f7a5-47c4-9e70-5958691ec5be. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1429.570574] env[67752]: DEBUG oslo_concurrency.lockutils [req-d0e3a657-7b89-4f6c-bcaa-a8d1eebef245 req-943b8b7f-de0d-4c03-86db-fc59412ea760 service nova] Acquiring lock "refresh_cache-e4ebb0f0-ab58-491f-bc4a-7548bbe415d8" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1429.570725] env[67752]: DEBUG oslo_concurrency.lockutils [req-d0e3a657-7b89-4f6c-bcaa-a8d1eebef245 req-943b8b7f-de0d-4c03-86db-fc59412ea760 service nova] Acquired lock "refresh_cache-e4ebb0f0-ab58-491f-bc4a-7548bbe415d8" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1429.570892] env[67752]: DEBUG nova.network.neutron [req-d0e3a657-7b89-4f6c-bcaa-a8d1eebef245 req-943b8b7f-de0d-4c03-86db-fc59412ea760 service nova] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Refreshing network info cache for port 19116fde-f7a5-47c4-9e70-5958691ec5be {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1430.031603] env[67752]: DEBUG nova.network.neutron [req-d0e3a657-7b89-4f6c-bcaa-a8d1eebef245 req-943b8b7f-de0d-4c03-86db-fc59412ea760 service nova] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Updated VIF entry in instance network info cache for port 19116fde-f7a5-47c4-9e70-5958691ec5be. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1430.032035] env[67752]: DEBUG nova.network.neutron [req-d0e3a657-7b89-4f6c-bcaa-a8d1eebef245 req-943b8b7f-de0d-4c03-86db-fc59412ea760 service nova] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Updating instance_info_cache with network_info: [{"id": "19116fde-f7a5-47c4-9e70-5958691ec5be", "address": "fa:16:3e:c7:a3:a4", "network": {"id": "9c1474f0-0949-46b4-a9a2-e00010321d55", "bridge": "br-int", "label": "tempest-ServersNegativeTestMultiTenantJSON-153947399-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.8", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d1cb577ba011491ba686908eea96d08f", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "8f40f5c4-c146-449c-884d-6f884dcf2acf", "external-id": "nsx-vlan-transportzone-240", "segmentation_id": 240, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap19116fde-f7", "ovs_interfaceid": "19116fde-f7a5-47c4-9e70-5958691ec5be", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1430.041617] env[67752]: DEBUG oslo_concurrency.lockutils [req-d0e3a657-7b89-4f6c-bcaa-a8d1eebef245 req-943b8b7f-de0d-4c03-86db-fc59412ea760 service nova] Releasing lock "refresh_cache-e4ebb0f0-ab58-491f-bc4a-7548bbe415d8" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1435.309147] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3f54ff6d-85e1-4d00-be13-bfc4a9353992 tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Acquiring lock "e4ebb0f0-ab58-491f-bc4a-7548bbe415d8" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1439.635480] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1439.635816] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Cleaning up deleted instances with incomplete migration {{(pid=67752) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11257}} [ 1441.333296] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "34c75d7e-e63e-4eff-afad-0edd61e023fc" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1441.333613] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "34c75d7e-e63e-4eff-afad-0edd61e023fc" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1441.639119] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1442.634925] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1443.634732] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1443.634913] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 1443.634965] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 1443.657138] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1443.657287] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1443.657426] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1443.657557] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: d3940c29-852a-427b-9027-aa4080150724] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1443.657761] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1443.657954] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1443.658158] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1443.658297] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1443.658422] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1443.658544] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1443.658665] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 1444.635362] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1444.635668] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1444.635746] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Cleaning up deleted instances {{(pid=67752) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11219}} [ 1444.650680] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] There are 1 instances to clean {{(pid=67752) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11228}} [ 1444.650867] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 7c8fc8d3-1f51-4793-8954-93451ebffa74] Instance has had 0 of 5 cleanup attempts {{(pid=67752) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11232}} [ 1444.898305] env[67752]: DEBUG oslo_concurrency.lockutils [None req-81605fee-f551-459c-a440-104dd375244b tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Acquiring lock "1227bf67-5435-4c53-86ac-224e7e00d86c" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1444.898555] env[67752]: DEBUG oslo_concurrency.lockutils [None req-81605fee-f551-459c-a440-104dd375244b tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Lock "1227bf67-5435-4c53-86ac-224e7e00d86c" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1446.675666] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1446.676025] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1447.634896] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1447.634896] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 1451.634647] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1451.634945] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1451.645949] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1451.646185] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1451.646354] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1451.646515] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1451.647698] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-620733fa-d071-47fb-8216-e5d11925e7f9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1451.657161] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7566715a-179e-45ca-9bf4-4360d516cdad {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1451.671670] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-916e3c16-1f33-44e7-b38e-1d2f3f96adcb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1451.679135] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-97db3d37-647f-4bd2-bf54-8fd571351220 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1451.709021] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181003MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1451.709122] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1451.709367] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1451.816185] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance b271cd73-a148-48d6-bed9-6a99512457bd actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1451.816310] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1451.816441] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f6499ee5-cb24-44e1-9fe7-c19feeb732eb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1451.816573] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d3940c29-852a-427b-9027-aa4080150724 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1451.816690] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1451.816808] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance a8bd310e-530a-46ab-add7-1b827ea5f399 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1451.816927] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9432cd04-c2df-428b-b31a-93d3ec0c9d1f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1451.817055] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 90e58033-eacd-433a-8d25-40f7d8403588 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1451.817175] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 81738fe6-6d50-4dfe-ae86-4c8771837168 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1451.817293] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e4ebb0f0-ab58-491f-bc4a-7548bbe415d8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1451.829949] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fe397d83-8625-4c5c-88b2-6c96c51c42bf has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1451.842076] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9ce8f114-c06d-4f08-b64a-980b94975ff7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1451.854413] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 5142518d-3b2a-474c-81d9-d658cb8467e3 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1451.869334] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c1975c17-15a5-4751-bf7f-d5b7e0d2fea1 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1451.880160] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1451.890426] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 7557e3a1-b298-401f-8320-5a48ddda3298 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1451.900525] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 34c75d7e-e63e-4eff-afad-0edd61e023fc has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1451.910708] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1227bf67-5435-4c53-86ac-224e7e00d86c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1451.910941] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1451.911107] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1452.126720] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6ee9d284-5445-4a53-ad8c-e9ab13ac7e8e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1452.134372] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b40f03c0-a15e-4497-b578-67370f32b2a6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1452.164738] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b8720bee-5ae2-4e9c-92ae-9f61f16a63a1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1452.174646] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7138dfd2-489f-4d0b-98f3-866b0430107c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1452.188777] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1452.198262] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1452.220738] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1452.221051] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.512s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1453.290250] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2362a7e3-7d07-4116-8026-618dc9901d50 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "214ac8b8-c04c-47cf-8dfd-fba169d6ec44" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1453.290808] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2362a7e3-7d07-4116-8026-618dc9901d50 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "214ac8b8-c04c-47cf-8dfd-fba169d6ec44" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1454.363596] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ac5a34f5-4287-4338-8801-fc5e30e196ac tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] Acquiring lock "7ab65bf3-7bbd-4c70-8a6d-18691bba92b5" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1454.363903] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ac5a34f5-4287-4338-8801-fc5e30e196ac tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] Lock "7ab65bf3-7bbd-4c70-8a6d-18691bba92b5" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1456.635130] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1460.093033] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4d9008c8-daab-45ba-856a-ef4fde4dc3c5 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Acquiring lock "048833ec-a7b2-4a8b-9204-788f2e9c5be7" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1460.093359] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4d9008c8-daab-45ba-856a-ef4fde4dc3c5 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Lock "048833ec-a7b2-4a8b-9204-788f2e9c5be7" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1472.215349] env[67752]: WARNING oslo_vmware.rw_handles [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1472.215349] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1472.215349] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1472.215349] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1472.215349] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1472.215349] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1472.215349] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1472.215349] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1472.215349] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1472.215349] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1472.215349] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1472.215349] env[67752]: ERROR oslo_vmware.rw_handles [ 1472.215953] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/bea47f13-4429-472c-943b-0b3645c0e181/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1472.217987] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1472.218253] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Copying Virtual Disk [datastore2] vmware_temp/bea47f13-4429-472c-943b-0b3645c0e181/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/bea47f13-4429-472c-943b-0b3645c0e181/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1472.218535] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-68857dbf-3d50-4651-a20f-2adbac1a2413 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1472.226112] env[67752]: DEBUG oslo_vmware.api [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Waiting for the task: (returnval){ [ 1472.226112] env[67752]: value = "task-3199782" [ 1472.226112] env[67752]: _type = "Task" [ 1472.226112] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1472.233969] env[67752]: DEBUG oslo_vmware.api [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Task: {'id': task-3199782, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1472.736430] env[67752]: DEBUG oslo_vmware.exceptions [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1472.736718] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1472.737292] env[67752]: ERROR nova.compute.manager [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1472.737292] env[67752]: Faults: ['InvalidArgument'] [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Traceback (most recent call last): [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] yield resources [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] self.driver.spawn(context, instance, image_meta, [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] self._fetch_image_if_missing(context, vi) [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] image_cache(vi, tmp_image_ds_loc) [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] vm_util.copy_virtual_disk( [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] session._wait_for_task(vmdk_copy_task) [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] return self.wait_for_task(task_ref) [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] return evt.wait() [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] result = hub.switch() [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] return self.greenlet.switch() [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] self.f(*self.args, **self.kw) [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] raise exceptions.translate_fault(task_info.error) [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Faults: ['InvalidArgument'] [ 1472.737292] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] [ 1472.738116] env[67752]: INFO nova.compute.manager [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Terminating instance [ 1472.739184] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1472.739392] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1472.739627] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-baa847f4-c2f5-4545-9c34-9d2ef4016ef5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1472.741790] env[67752]: DEBUG nova.compute.manager [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1472.741986] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1472.742690] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ba6d1700-a837-4c73-8895-8efd0cd1f4db {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1472.750199] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1472.750433] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-37c3f330-6c04-44b5-9d12-00b8e577595d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1472.752517] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1472.752689] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1472.753615] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c047a6d9-c053-46a6-9d99-de2c99eecd1f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1472.758169] env[67752]: DEBUG oslo_vmware.api [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Waiting for the task: (returnval){ [ 1472.758169] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]526b15e6-4b11-ca03-155f-d5448634bdb6" [ 1472.758169] env[67752]: _type = "Task" [ 1472.758169] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1472.769336] env[67752]: DEBUG oslo_vmware.api [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]526b15e6-4b11-ca03-155f-d5448634bdb6, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1472.824498] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1472.824714] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1472.824901] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Deleting the datastore file [datastore2] b271cd73-a148-48d6-bed9-6a99512457bd {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1472.825219] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-8341640e-1e3a-41e4-a4d5-ab658b807acb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1472.830669] env[67752]: DEBUG oslo_vmware.api [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Waiting for the task: (returnval){ [ 1472.830669] env[67752]: value = "task-3199784" [ 1472.830669] env[67752]: _type = "Task" [ 1472.830669] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1472.838009] env[67752]: DEBUG oslo_vmware.api [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Task: {'id': task-3199784, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1473.269028] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1473.269362] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Creating directory with path [datastore2] vmware_temp/d333788e-bd77-4639-90dd-b937808ba00c/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1473.269552] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-cdcd5e06-a1bc-48ee-8c5c-adfcfa0407ad {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1473.280662] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Created directory with path [datastore2] vmware_temp/d333788e-bd77-4639-90dd-b937808ba00c/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1473.280865] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Fetch image to [datastore2] vmware_temp/d333788e-bd77-4639-90dd-b937808ba00c/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1473.281058] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/d333788e-bd77-4639-90dd-b937808ba00c/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1473.281772] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6966ecc8-c8b0-4e72-9469-e1711410b7b0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1473.288164] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-747f0658-805c-44e4-9375-eaa52e8d2ff4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1473.297364] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dfa0cf24-b433-4818-8798-8828a8333dfe {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1473.326813] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d094913-87ae-495c-8778-18ffe0a6feec {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1473.334514] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-092b9e5a-cffb-4e6e-a1f6-0e29cbbf8244 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1473.340555] env[67752]: DEBUG oslo_vmware.api [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Task: {'id': task-3199784, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.062637} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1473.340555] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1473.340710] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1473.340900] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1473.341094] env[67752]: INFO nova.compute.manager [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1473.343978] env[67752]: DEBUG nova.compute.claims [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1473.344170] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1473.344385] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1473.358719] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1473.417889] env[67752]: DEBUG oslo_vmware.rw_handles [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/d333788e-bd77-4639-90dd-b937808ba00c/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1473.481369] env[67752]: DEBUG oslo_vmware.rw_handles [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1473.481607] env[67752]: DEBUG oslo_vmware.rw_handles [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Closing write handle for https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/d333788e-bd77-4639-90dd-b937808ba00c/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1473.659815] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cd014303-4539-47e6-94cd-78aa2b5947e1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1473.668582] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-34e91760-2b18-45a1-90c6-4f68aeacad78 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1473.698130] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54e90420-85bc-45ad-a3fb-15f4420f1667 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1473.705238] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7ba9e33-1d01-409e-94dc-b56bddd1b183 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1473.717972] env[67752]: DEBUG nova.compute.provider_tree [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1473.726716] env[67752]: DEBUG nova.scheduler.client.report [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1473.743054] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.398s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1473.743604] env[67752]: ERROR nova.compute.manager [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1473.743604] env[67752]: Faults: ['InvalidArgument'] [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Traceback (most recent call last): [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] self.driver.spawn(context, instance, image_meta, [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] self._fetch_image_if_missing(context, vi) [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] image_cache(vi, tmp_image_ds_loc) [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] vm_util.copy_virtual_disk( [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] session._wait_for_task(vmdk_copy_task) [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] return self.wait_for_task(task_ref) [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] return evt.wait() [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] result = hub.switch() [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] return self.greenlet.switch() [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] self.f(*self.args, **self.kw) [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] raise exceptions.translate_fault(task_info.error) [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Faults: ['InvalidArgument'] [ 1473.743604] env[67752]: ERROR nova.compute.manager [instance: b271cd73-a148-48d6-bed9-6a99512457bd] [ 1473.744373] env[67752]: DEBUG nova.compute.utils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1473.746249] env[67752]: DEBUG nova.compute.manager [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Build of instance b271cd73-a148-48d6-bed9-6a99512457bd was re-scheduled: A specified parameter was not correct: fileType [ 1473.746249] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1473.746631] env[67752]: DEBUG nova.compute.manager [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1473.746807] env[67752]: DEBUG nova.compute.manager [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1473.746981] env[67752]: DEBUG nova.compute.manager [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1473.747170] env[67752]: DEBUG nova.network.neutron [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1474.234381] env[67752]: DEBUG nova.network.neutron [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1474.249977] env[67752]: INFO nova.compute.manager [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Took 0.50 seconds to deallocate network for instance. [ 1474.347954] env[67752]: INFO nova.scheduler.client.report [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Deleted allocations for instance b271cd73-a148-48d6-bed9-6a99512457bd [ 1474.370299] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33cc056f-db8d-4fda-8b89-b23f0b172b94 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Lock "b271cd73-a148-48d6-bed9-6a99512457bd" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 632.956s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1474.371640] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8b4be1c0-6841-4ccd-a60c-9978f6ce6b69 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Lock "b271cd73-a148-48d6-bed9-6a99512457bd" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 435.504s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1474.371906] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8b4be1c0-6841-4ccd-a60c-9978f6ce6b69 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Acquiring lock "b271cd73-a148-48d6-bed9-6a99512457bd-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1474.372194] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8b4be1c0-6841-4ccd-a60c-9978f6ce6b69 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Lock "b271cd73-a148-48d6-bed9-6a99512457bd-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1474.372403] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8b4be1c0-6841-4ccd-a60c-9978f6ce6b69 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Lock "b271cd73-a148-48d6-bed9-6a99512457bd-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1474.374596] env[67752]: INFO nova.compute.manager [None req-8b4be1c0-6841-4ccd-a60c-9978f6ce6b69 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Terminating instance [ 1474.376459] env[67752]: DEBUG nova.compute.manager [None req-8b4be1c0-6841-4ccd-a60c-9978f6ce6b69 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1474.376676] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-8b4be1c0-6841-4ccd-a60c-9978f6ce6b69 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1474.377205] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ca61e40f-8bc8-475a-8853-24ac06e029d5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1474.387474] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-05038759-61ff-45ee-ad16-c2a08d3e15f1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1474.400815] env[67752]: DEBUG nova.compute.manager [None req-d109a272-e6be-47f6-9b20-86e0f90c6dc3 tempest-ServerRescueNegativeTestJSON-761915415 tempest-ServerRescueNegativeTestJSON-761915415-project-member] [instance: ba83b143-d752-4166-8ce0-fbd7fd4875a4] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1474.422225] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-8b4be1c0-6841-4ccd-a60c-9978f6ce6b69 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance b271cd73-a148-48d6-bed9-6a99512457bd could not be found. [ 1474.422454] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-8b4be1c0-6841-4ccd-a60c-9978f6ce6b69 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1474.422640] env[67752]: INFO nova.compute.manager [None req-8b4be1c0-6841-4ccd-a60c-9978f6ce6b69 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Took 0.05 seconds to destroy the instance on the hypervisor. [ 1474.422916] env[67752]: DEBUG oslo.service.loopingcall [None req-8b4be1c0-6841-4ccd-a60c-9978f6ce6b69 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1474.423185] env[67752]: DEBUG nova.compute.manager [-] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1474.423280] env[67752]: DEBUG nova.network.neutron [-] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1474.425960] env[67752]: DEBUG nova.compute.manager [None req-d109a272-e6be-47f6-9b20-86e0f90c6dc3 tempest-ServerRescueNegativeTestJSON-761915415 tempest-ServerRescueNegativeTestJSON-761915415-project-member] [instance: ba83b143-d752-4166-8ce0-fbd7fd4875a4] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1474.449675] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d109a272-e6be-47f6-9b20-86e0f90c6dc3 tempest-ServerRescueNegativeTestJSON-761915415 tempest-ServerRescueNegativeTestJSON-761915415-project-member] Lock "ba83b143-d752-4166-8ce0-fbd7fd4875a4" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 222.040s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1474.451924] env[67752]: DEBUG nova.network.neutron [-] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1474.463023] env[67752]: INFO nova.compute.manager [-] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] Took 0.04 seconds to deallocate network for instance. [ 1474.468823] env[67752]: DEBUG nova.compute.manager [None req-ea5f7110-782b-434d-b98d-3b6e08eddc53 tempest-ServerRescueNegativeTestJSON-761915415 tempest-ServerRescueNegativeTestJSON-761915415-project-member] [instance: 6000d3ce-6bdc-4683-a1a9-fbc1e447aad4] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1474.492610] env[67752]: DEBUG nova.compute.manager [None req-ea5f7110-782b-434d-b98d-3b6e08eddc53 tempest-ServerRescueNegativeTestJSON-761915415 tempest-ServerRescueNegativeTestJSON-761915415-project-member] [instance: 6000d3ce-6bdc-4683-a1a9-fbc1e447aad4] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1474.513085] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5f7110-782b-434d-b98d-3b6e08eddc53 tempest-ServerRescueNegativeTestJSON-761915415 tempest-ServerRescueNegativeTestJSON-761915415-project-member] Lock "6000d3ce-6bdc-4683-a1a9-fbc1e447aad4" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 221.593s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1474.527504] env[67752]: DEBUG nova.compute.manager [None req-12b68195-bd71-4a5b-9309-cb555652ea5f tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: fe397d83-8625-4c5c-88b2-6c96c51c42bf] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1474.563660] env[67752]: DEBUG nova.compute.manager [None req-12b68195-bd71-4a5b-9309-cb555652ea5f tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: fe397d83-8625-4c5c-88b2-6c96c51c42bf] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1474.568501] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8b4be1c0-6841-4ccd-a60c-9978f6ce6b69 tempest-ServerMetadataNegativeTestJSON-269209030 tempest-ServerMetadataNegativeTestJSON-269209030-project-member] Lock "b271cd73-a148-48d6-bed9-6a99512457bd" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.197s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1474.570280] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "b271cd73-a148-48d6-bed9-6a99512457bd" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 294.928s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1474.570280] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: b271cd73-a148-48d6-bed9-6a99512457bd] During sync_power_state the instance has a pending task (deleting). Skip. [ 1474.570280] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "b271cd73-a148-48d6-bed9-6a99512457bd" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1474.589218] env[67752]: DEBUG oslo_concurrency.lockutils [None req-12b68195-bd71-4a5b-9309-cb555652ea5f tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "fe397d83-8625-4c5c-88b2-6c96c51c42bf" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 218.568s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1474.599099] env[67752]: DEBUG nova.compute.manager [None req-ae68920d-b3c2-4b1b-a8f3-c069c02ae72d tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] [instance: 9ce8f114-c06d-4f08-b64a-980b94975ff7] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1474.621911] env[67752]: DEBUG nova.compute.manager [None req-ae68920d-b3c2-4b1b-a8f3-c069c02ae72d tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] [instance: 9ce8f114-c06d-4f08-b64a-980b94975ff7] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1474.641628] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ae68920d-b3c2-4b1b-a8f3-c069c02ae72d tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] Lock "9ce8f114-c06d-4f08-b64a-980b94975ff7" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 216.461s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1474.649782] env[67752]: DEBUG nova.compute.manager [None req-3e9cfa33-5dce-43aa-b42f-3b71ffa58a33 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: 5142518d-3b2a-474c-81d9-d658cb8467e3] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1474.674159] env[67752]: DEBUG nova.compute.manager [None req-3e9cfa33-5dce-43aa-b42f-3b71ffa58a33 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: 5142518d-3b2a-474c-81d9-d658cb8467e3] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1474.695343] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3e9cfa33-5dce-43aa-b42f-3b71ffa58a33 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Lock "5142518d-3b2a-474c-81d9-d658cb8467e3" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 212.746s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1474.704038] env[67752]: DEBUG nova.compute.manager [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1474.756582] env[67752]: DEBUG oslo_concurrency.lockutils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1474.756846] env[67752]: DEBUG oslo_concurrency.lockutils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1474.758412] env[67752]: INFO nova.compute.claims [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1475.013047] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a05a6785-6885-4333-baa1-7fac09b83f12 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1475.020472] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-961569e9-16ec-4e98-b306-cbc789b2c339 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1475.050424] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c034b902-f636-40f9-b650-453ccd542fe4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1475.057216] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-85903548-59f9-47c4-9219-9db94627e896 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1475.069863] env[67752]: DEBUG nova.compute.provider_tree [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1475.078974] env[67752]: DEBUG nova.scheduler.client.report [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1475.092338] env[67752]: DEBUG oslo_concurrency.lockutils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.335s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1475.092954] env[67752]: DEBUG nova.compute.manager [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1475.127970] env[67752]: DEBUG nova.compute.utils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1475.129686] env[67752]: DEBUG nova.compute.manager [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1475.129861] env[67752]: DEBUG nova.network.neutron [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1475.137860] env[67752]: DEBUG nova.compute.manager [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1475.201470] env[67752]: DEBUG nova.compute.manager [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1475.213050] env[67752]: DEBUG nova.policy [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '639cb98e283640c3907a927d4c10c359', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'dc0e175514604a85bb7d9284ff4be671', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1475.232256] env[67752]: DEBUG nova.virt.hardware [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1475.232515] env[67752]: DEBUG nova.virt.hardware [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1475.232676] env[67752]: DEBUG nova.virt.hardware [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1475.232895] env[67752]: DEBUG nova.virt.hardware [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1475.233485] env[67752]: DEBUG nova.virt.hardware [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1475.233485] env[67752]: DEBUG nova.virt.hardware [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1475.233485] env[67752]: DEBUG nova.virt.hardware [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1475.233675] env[67752]: DEBUG nova.virt.hardware [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1475.233721] env[67752]: DEBUG nova.virt.hardware [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1475.233937] env[67752]: DEBUG nova.virt.hardware [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1475.234148] env[67752]: DEBUG nova.virt.hardware [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1475.234989] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e06c58aa-29cd-40f2-a06e-ac0fbb0024b1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1475.243075] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0cbc6246-0c5b-426a-a646-6c9c5b2138b5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1475.574364] env[67752]: DEBUG nova.network.neutron [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Successfully created port: d2fc6c48-3751-4612-a162-14c678b7edd4 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1476.320249] env[67752]: DEBUG nova.compute.manager [req-8e51a73d-6d2e-4faa-b246-6daac0bb5019 req-af7774b4-ea8a-4063-a4f1-38ef8bd11d66 service nova] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Received event network-vif-plugged-d2fc6c48-3751-4612-a162-14c678b7edd4 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1476.320249] env[67752]: DEBUG oslo_concurrency.lockutils [req-8e51a73d-6d2e-4faa-b246-6daac0bb5019 req-af7774b4-ea8a-4063-a4f1-38ef8bd11d66 service nova] Acquiring lock "c1975c17-15a5-4751-bf7f-d5b7e0d2fea1-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1476.320249] env[67752]: DEBUG oslo_concurrency.lockutils [req-8e51a73d-6d2e-4faa-b246-6daac0bb5019 req-af7774b4-ea8a-4063-a4f1-38ef8bd11d66 service nova] Lock "c1975c17-15a5-4751-bf7f-d5b7e0d2fea1-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1476.320249] env[67752]: DEBUG oslo_concurrency.lockutils [req-8e51a73d-6d2e-4faa-b246-6daac0bb5019 req-af7774b4-ea8a-4063-a4f1-38ef8bd11d66 service nova] Lock "c1975c17-15a5-4751-bf7f-d5b7e0d2fea1-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1476.320249] env[67752]: DEBUG nova.compute.manager [req-8e51a73d-6d2e-4faa-b246-6daac0bb5019 req-af7774b4-ea8a-4063-a4f1-38ef8bd11d66 service nova] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] No waiting events found dispatching network-vif-plugged-d2fc6c48-3751-4612-a162-14c678b7edd4 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1476.320249] env[67752]: WARNING nova.compute.manager [req-8e51a73d-6d2e-4faa-b246-6daac0bb5019 req-af7774b4-ea8a-4063-a4f1-38ef8bd11d66 service nova] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Received unexpected event network-vif-plugged-d2fc6c48-3751-4612-a162-14c678b7edd4 for instance with vm_state building and task_state spawning. [ 1476.469617] env[67752]: DEBUG nova.network.neutron [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Successfully updated port: d2fc6c48-3751-4612-a162-14c678b7edd4 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1476.481810] env[67752]: DEBUG oslo_concurrency.lockutils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquiring lock "refresh_cache-c1975c17-15a5-4751-bf7f-d5b7e0d2fea1" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1476.481810] env[67752]: DEBUG oslo_concurrency.lockutils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquired lock "refresh_cache-c1975c17-15a5-4751-bf7f-d5b7e0d2fea1" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1476.481810] env[67752]: DEBUG nova.network.neutron [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1476.539691] env[67752]: DEBUG nova.network.neutron [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1476.704627] env[67752]: DEBUG nova.network.neutron [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Updating instance_info_cache with network_info: [{"id": "d2fc6c48-3751-4612-a162-14c678b7edd4", "address": "fa:16:3e:3c:97:24", "network": {"id": "b684aefa-3309-4e2f-a55b-32853fdd1a0d", "bridge": "br-int", "label": "tempest-AttachVolumeTestJSON-1831973259-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "dc0e175514604a85bb7d9284ff4be671", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "257e5ea7-8b80-4301-9900-a754f1fe2031", "external-id": "nsx-vlan-transportzone-682", "segmentation_id": 682, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapd2fc6c48-37", "ovs_interfaceid": "d2fc6c48-3751-4612-a162-14c678b7edd4", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1476.715752] env[67752]: DEBUG oslo_concurrency.lockutils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Releasing lock "refresh_cache-c1975c17-15a5-4751-bf7f-d5b7e0d2fea1" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1476.716069] env[67752]: DEBUG nova.compute.manager [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Instance network_info: |[{"id": "d2fc6c48-3751-4612-a162-14c678b7edd4", "address": "fa:16:3e:3c:97:24", "network": {"id": "b684aefa-3309-4e2f-a55b-32853fdd1a0d", "bridge": "br-int", "label": "tempest-AttachVolumeTestJSON-1831973259-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "dc0e175514604a85bb7d9284ff4be671", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "257e5ea7-8b80-4301-9900-a754f1fe2031", "external-id": "nsx-vlan-transportzone-682", "segmentation_id": 682, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapd2fc6c48-37", "ovs_interfaceid": "d2fc6c48-3751-4612-a162-14c678b7edd4", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1476.716479] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:3c:97:24', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '257e5ea7-8b80-4301-9900-a754f1fe2031', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'd2fc6c48-3751-4612-a162-14c678b7edd4', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1476.723717] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Creating folder: Project (dc0e175514604a85bb7d9284ff4be671). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1476.724235] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-d9137c54-01f2-4e96-9947-9f0cebf315a3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1476.734519] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Created folder: Project (dc0e175514604a85bb7d9284ff4be671) in parent group-v639722. [ 1476.734691] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Creating folder: Instances. Parent ref: group-v639818. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1476.734892] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-9a40d0c8-e3de-42c6-b637-c533c8479d4a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1476.742834] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Created folder: Instances in parent group-v639818. [ 1476.743073] env[67752]: DEBUG oslo.service.loopingcall [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1476.743250] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1476.743425] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-775ea1ab-e34a-4485-9dd7-e35c61123de1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1476.760736] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1476.760736] env[67752]: value = "task-3199787" [ 1476.760736] env[67752]: _type = "Task" [ 1476.760736] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1476.767408] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199787, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1477.270635] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199787, 'name': CreateVM_Task, 'duration_secs': 0.281008} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1477.270839] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1477.271516] env[67752]: DEBUG oslo_concurrency.lockutils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1477.271682] env[67752]: DEBUG oslo_concurrency.lockutils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1477.272020] env[67752]: DEBUG oslo_concurrency.lockutils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1477.272274] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-6a3defd9-8c2e-41f1-9090-caa67b781f5b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1477.276613] env[67752]: DEBUG oslo_vmware.api [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Waiting for the task: (returnval){ [ 1477.276613] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52ec1436-37b2-ea58-1d7e-783b92a6f197" [ 1477.276613] env[67752]: _type = "Task" [ 1477.276613] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1477.284103] env[67752]: DEBUG oslo_vmware.api [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52ec1436-37b2-ea58-1d7e-783b92a6f197, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1477.787324] env[67752]: DEBUG oslo_concurrency.lockutils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1477.787660] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1477.787722] env[67752]: DEBUG oslo_concurrency.lockutils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1478.421802] env[67752]: DEBUG nova.compute.manager [req-f3ad73c7-db52-4735-a3e4-dcb0e2e201b4 req-1f426bf3-22e4-40a4-9ca3-c8e1f47139a6 service nova] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Received event network-changed-d2fc6c48-3751-4612-a162-14c678b7edd4 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1478.422012] env[67752]: DEBUG nova.compute.manager [req-f3ad73c7-db52-4735-a3e4-dcb0e2e201b4 req-1f426bf3-22e4-40a4-9ca3-c8e1f47139a6 service nova] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Refreshing instance network info cache due to event network-changed-d2fc6c48-3751-4612-a162-14c678b7edd4. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1478.422240] env[67752]: DEBUG oslo_concurrency.lockutils [req-f3ad73c7-db52-4735-a3e4-dcb0e2e201b4 req-1f426bf3-22e4-40a4-9ca3-c8e1f47139a6 service nova] Acquiring lock "refresh_cache-c1975c17-15a5-4751-bf7f-d5b7e0d2fea1" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1478.422384] env[67752]: DEBUG oslo_concurrency.lockutils [req-f3ad73c7-db52-4735-a3e4-dcb0e2e201b4 req-1f426bf3-22e4-40a4-9ca3-c8e1f47139a6 service nova] Acquired lock "refresh_cache-c1975c17-15a5-4751-bf7f-d5b7e0d2fea1" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1478.422542] env[67752]: DEBUG nova.network.neutron [req-f3ad73c7-db52-4735-a3e4-dcb0e2e201b4 req-1f426bf3-22e4-40a4-9ca3-c8e1f47139a6 service nova] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Refreshing network info cache for port d2fc6c48-3751-4612-a162-14c678b7edd4 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1478.814873] env[67752]: DEBUG nova.network.neutron [req-f3ad73c7-db52-4735-a3e4-dcb0e2e201b4 req-1f426bf3-22e4-40a4-9ca3-c8e1f47139a6 service nova] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Updated VIF entry in instance network info cache for port d2fc6c48-3751-4612-a162-14c678b7edd4. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1478.815293] env[67752]: DEBUG nova.network.neutron [req-f3ad73c7-db52-4735-a3e4-dcb0e2e201b4 req-1f426bf3-22e4-40a4-9ca3-c8e1f47139a6 service nova] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Updating instance_info_cache with network_info: [{"id": "d2fc6c48-3751-4612-a162-14c678b7edd4", "address": "fa:16:3e:3c:97:24", "network": {"id": "b684aefa-3309-4e2f-a55b-32853fdd1a0d", "bridge": "br-int", "label": "tempest-AttachVolumeTestJSON-1831973259-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "dc0e175514604a85bb7d9284ff4be671", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "257e5ea7-8b80-4301-9900-a754f1fe2031", "external-id": "nsx-vlan-transportzone-682", "segmentation_id": 682, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapd2fc6c48-37", "ovs_interfaceid": "d2fc6c48-3751-4612-a162-14c678b7edd4", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1478.824183] env[67752]: DEBUG oslo_concurrency.lockutils [req-f3ad73c7-db52-4735-a3e4-dcb0e2e201b4 req-1f426bf3-22e4-40a4-9ca3-c8e1f47139a6 service nova] Releasing lock "refresh_cache-c1975c17-15a5-4751-bf7f-d5b7e0d2fea1" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1501.638064] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1502.634840] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1504.631269] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1505.634655] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1505.634929] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 1505.634982] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 1505.655520] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1505.655702] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1505.655833] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: d3940c29-852a-427b-9027-aa4080150724] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1505.655962] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1505.656106] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1505.656233] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1505.656374] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1505.656502] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1505.656623] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1505.656747] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1505.656870] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 1506.634536] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1506.635602] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1507.636106] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1507.636106] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1507.636489] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 1511.635887] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1512.636037] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1512.647735] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1512.647971] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1512.648164] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1512.648323] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1512.649476] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d828e833-5a86-4574-b086-0f920bec7ccc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1512.658980] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-72d4d604-9c4f-4db1-9818-2950ae6385b2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1512.672659] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-33dd5361-5e8a-427c-81f3-67451ca70702 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1512.679312] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e371342d-280e-40af-9fa7-49bd7d82ed68 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1512.707191] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181004MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1512.707362] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1512.707533] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1512.781238] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1512.781411] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f6499ee5-cb24-44e1-9fe7-c19feeb732eb actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1512.781545] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d3940c29-852a-427b-9027-aa4080150724 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1512.781674] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1512.781860] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance a8bd310e-530a-46ab-add7-1b827ea5f399 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1512.782017] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9432cd04-c2df-428b-b31a-93d3ec0c9d1f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1512.782149] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 90e58033-eacd-433a-8d25-40f7d8403588 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1512.782268] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 81738fe6-6d50-4dfe-ae86-4c8771837168 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1512.782384] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e4ebb0f0-ab58-491f-bc4a-7548bbe415d8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1512.782501] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c1975c17-15a5-4751-bf7f-d5b7e0d2fea1 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1512.794040] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1512.804209] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 7557e3a1-b298-401f-8320-5a48ddda3298 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1512.813584] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 34c75d7e-e63e-4eff-afad-0edd61e023fc has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1512.822662] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1227bf67-5435-4c53-86ac-224e7e00d86c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1512.831700] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 214ac8b8-c04c-47cf-8dfd-fba169d6ec44 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1512.840556] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 7ab65bf3-7bbd-4c70-8a6d-18691bba92b5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1512.849555] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 048833ec-a7b2-4a8b-9204-788f2e9c5be7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1512.849831] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1512.850066] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1513.035076] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9529baaa-4583-478f-90f5-c18cd39b7eb2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1513.042326] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a82ada91-08cd-4156-9327-f8e2dee29d54 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1513.072793] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c93cbb3c-dda4-4e41-a75e-3b635030320b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1513.079666] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-02aad604-a73d-4148-9f50-eeda70d7e452 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1513.091988] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1513.100198] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1513.112878] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1513.113077] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.406s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1518.708097] env[67752]: WARNING oslo_vmware.rw_handles [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1518.708097] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1518.708097] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1518.708097] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1518.708097] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1518.708097] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1518.708097] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1518.708097] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1518.708097] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1518.708097] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1518.708097] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1518.708097] env[67752]: ERROR oslo_vmware.rw_handles [ 1518.708733] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/d333788e-bd77-4639-90dd-b937808ba00c/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1518.711116] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1518.711391] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Copying Virtual Disk [datastore2] vmware_temp/d333788e-bd77-4639-90dd-b937808ba00c/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/d333788e-bd77-4639-90dd-b937808ba00c/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1518.711720] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-1dd9f1c1-fae0-47c8-8155-d3ce44fecc3b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1518.719229] env[67752]: DEBUG oslo_vmware.api [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Waiting for the task: (returnval){ [ 1518.719229] env[67752]: value = "task-3199788" [ 1518.719229] env[67752]: _type = "Task" [ 1518.719229] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1518.726796] env[67752]: DEBUG oslo_vmware.api [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Task: {'id': task-3199788, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1519.230231] env[67752]: DEBUG oslo_vmware.exceptions [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1519.230506] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1519.231080] env[67752]: ERROR nova.compute.manager [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1519.231080] env[67752]: Faults: ['InvalidArgument'] [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Traceback (most recent call last): [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] yield resources [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] self.driver.spawn(context, instance, image_meta, [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] self._fetch_image_if_missing(context, vi) [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] image_cache(vi, tmp_image_ds_loc) [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] vm_util.copy_virtual_disk( [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] session._wait_for_task(vmdk_copy_task) [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] return self.wait_for_task(task_ref) [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] return evt.wait() [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] result = hub.switch() [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] return self.greenlet.switch() [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] self.f(*self.args, **self.kw) [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] raise exceptions.translate_fault(task_info.error) [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Faults: ['InvalidArgument'] [ 1519.231080] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] [ 1519.231913] env[67752]: INFO nova.compute.manager [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Terminating instance [ 1519.232971] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1519.233192] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1519.233954] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-fadb7d35-7bfc-4fcc-a387-7eba14d87813 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1519.235718] env[67752]: DEBUG nova.compute.manager [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1519.236029] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1519.237200] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a527cb4b-cd12-4285-a0b3-9c2f6f326ccb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1519.246874] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1519.248149] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-c4a6828d-8400-4123-9938-762e4f0e7a40 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1519.249862] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1519.250093] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1519.250821] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-eaa67ea4-48b0-44f2-aa05-58bca543bd3a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1519.255965] env[67752]: DEBUG oslo_vmware.api [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Waiting for the task: (returnval){ [ 1519.255965] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5234e6db-09ff-261d-a0c1-c4650d4f183a" [ 1519.255965] env[67752]: _type = "Task" [ 1519.255965] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1519.264027] env[67752]: DEBUG oslo_vmware.api [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5234e6db-09ff-261d-a0c1-c4650d4f183a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1519.319040] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1519.319414] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1519.319522] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Deleting the datastore file [datastore2] 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1519.319721] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-bcffcd14-d3f8-43ea-bafe-82b24406f300 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1519.326082] env[67752]: DEBUG oslo_vmware.api [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Waiting for the task: (returnval){ [ 1519.326082] env[67752]: value = "task-3199790" [ 1519.326082] env[67752]: _type = "Task" [ 1519.326082] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1519.334058] env[67752]: DEBUG oslo_vmware.api [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Task: {'id': task-3199790, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1519.765989] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1519.766347] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Creating directory with path [datastore2] vmware_temp/e922adc0-f8d9-4116-8063-c8180f526013/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1519.766488] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-ced72f17-1b2e-4c71-9799-61290a23239a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1519.777423] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Created directory with path [datastore2] vmware_temp/e922adc0-f8d9-4116-8063-c8180f526013/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1519.777602] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Fetch image to [datastore2] vmware_temp/e922adc0-f8d9-4116-8063-c8180f526013/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1519.777792] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/e922adc0-f8d9-4116-8063-c8180f526013/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1519.778523] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b574fda0-4f2e-4f4e-92d0-6ce99b429d8c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1519.784666] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7a7251eb-46e2-417b-9388-76ac8890a335 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1519.793281] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d6fbe471-23fe-4c85-9641-d706ca8a71a7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1519.823738] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d888a5a-926f-4516-ae3a-1a6f07db485d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1519.831434] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-2eeb50e9-0b44-4f46-92b4-ed4d097e2c3f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1519.835639] env[67752]: DEBUG oslo_vmware.api [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Task: {'id': task-3199790, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.0743} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1519.836215] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1519.836426] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1519.836650] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1519.836835] env[67752]: INFO nova.compute.manager [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1519.838957] env[67752]: DEBUG nova.compute.claims [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1519.839151] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1519.839365] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1519.854147] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1520.012349] env[67752]: DEBUG oslo_vmware.rw_handles [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/e922adc0-f8d9-4116-8063-c8180f526013/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1520.074635] env[67752]: DEBUG oslo_vmware.rw_handles [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1520.074958] env[67752]: DEBUG oslo_vmware.rw_handles [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/e922adc0-f8d9-4116-8063-c8180f526013/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1520.124322] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3afcbfb-bc14-44ea-b4e1-43e01b184e57 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1520.131937] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3830e93-8ae0-4725-aca4-7b07f2e27337 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1520.160274] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa52bff9-2b70-4c6e-8929-7eaf417388f0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1520.166639] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-94c24c49-f21e-45f7-aad3-73ab0b01880e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1520.178910] env[67752]: DEBUG nova.compute.provider_tree [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1520.186904] env[67752]: DEBUG nova.scheduler.client.report [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1520.201481] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.362s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1520.201992] env[67752]: ERROR nova.compute.manager [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1520.201992] env[67752]: Faults: ['InvalidArgument'] [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Traceback (most recent call last): [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] self.driver.spawn(context, instance, image_meta, [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] self._fetch_image_if_missing(context, vi) [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] image_cache(vi, tmp_image_ds_loc) [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] vm_util.copy_virtual_disk( [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] session._wait_for_task(vmdk_copy_task) [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] return self.wait_for_task(task_ref) [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] return evt.wait() [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] result = hub.switch() [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] return self.greenlet.switch() [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] self.f(*self.args, **self.kw) [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] raise exceptions.translate_fault(task_info.error) [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Faults: ['InvalidArgument'] [ 1520.201992] env[67752]: ERROR nova.compute.manager [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] [ 1520.202853] env[67752]: DEBUG nova.compute.utils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1520.204030] env[67752]: DEBUG nova.compute.manager [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Build of instance 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9 was re-scheduled: A specified parameter was not correct: fileType [ 1520.204030] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1520.204400] env[67752]: DEBUG nova.compute.manager [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1520.204572] env[67752]: DEBUG nova.compute.manager [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1520.204740] env[67752]: DEBUG nova.compute.manager [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1520.204907] env[67752]: DEBUG nova.network.neutron [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1521.023574] env[67752]: DEBUG nova.network.neutron [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1521.035184] env[67752]: INFO nova.compute.manager [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Took 0.83 seconds to deallocate network for instance. [ 1521.179752] env[67752]: INFO nova.scheduler.client.report [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Deleted allocations for instance 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9 [ 1521.202450] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d84ad651-80e0-4ea6-8e33-0f9e70bccdfb tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Lock "3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 626.971s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1521.203836] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4094cbfb-3682-4575-957c-d8b65674b023 tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Lock "3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 430.285s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1521.204471] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4094cbfb-3682-4575-957c-d8b65674b023 tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Acquiring lock "3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1521.204471] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4094cbfb-3682-4575-957c-d8b65674b023 tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Lock "3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1521.204471] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4094cbfb-3682-4575-957c-d8b65674b023 tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Lock "3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1521.207169] env[67752]: INFO nova.compute.manager [None req-4094cbfb-3682-4575-957c-d8b65674b023 tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Terminating instance [ 1521.209076] env[67752]: DEBUG nova.compute.manager [None req-4094cbfb-3682-4575-957c-d8b65674b023 tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1521.209281] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-4094cbfb-3682-4575-957c-d8b65674b023 tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1521.209768] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-fb0ee6ac-a844-4450-a77b-f9e5df1cbc59 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1521.219513] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-331eaff6-f81c-43f7-966a-86343bf859f3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1521.231610] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1521.252638] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-4094cbfb-3682-4575-957c-d8b65674b023 tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9 could not be found. [ 1521.252794] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-4094cbfb-3682-4575-957c-d8b65674b023 tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1521.253027] env[67752]: INFO nova.compute.manager [None req-4094cbfb-3682-4575-957c-d8b65674b023 tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1521.253299] env[67752]: DEBUG oslo.service.loopingcall [None req-4094cbfb-3682-4575-957c-d8b65674b023 tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1521.253650] env[67752]: DEBUG nova.compute.manager [-] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1521.253818] env[67752]: DEBUG nova.network.neutron [-] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1521.279253] env[67752]: DEBUG nova.network.neutron [-] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1521.281056] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1521.281198] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1521.282755] env[67752]: INFO nova.compute.claims [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1521.287887] env[67752]: INFO nova.compute.manager [-] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] Took 0.03 seconds to deallocate network for instance. [ 1521.383011] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4094cbfb-3682-4575-957c-d8b65674b023 tempest-InstanceActionsV221TestJSON-211010463 tempest-InstanceActionsV221TestJSON-211010463-project-member] Lock "3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.179s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1521.384414] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 341.743s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1521.384601] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9] During sync_power_state the instance has a pending task (deleting). Skip. [ 1521.384851] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "3f9f475b-6f6e-41b8-8f3b-e5dab74d99f9" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1521.548744] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a45ace99-7a6a-4bde-a954-9e5b55c32cf9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1521.556709] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d2ed7110-f3c1-438b-9095-28161c63cff9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1521.588805] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3889802f-bfd1-4e04-a0b2-966a1cb8acd4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1521.596547] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-572b1c12-670d-43d5-a10c-5ea90c6341d3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1521.609789] env[67752]: DEBUG nova.compute.provider_tree [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1521.623962] env[67752]: DEBUG nova.scheduler.client.report [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1521.640470] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.359s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1521.641074] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1521.688299] env[67752]: DEBUG nova.compute.utils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1521.689702] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1521.689884] env[67752]: DEBUG nova.network.neutron [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1521.726097] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1521.765232] env[67752]: DEBUG nova.policy [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '827cdef2d9b647f79e05f8d23696a090', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd6b1ff6c8aaa49bd968d23a1fe068529', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1521.795379] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1521.824309] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1521.824563] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1521.824734] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1521.824913] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1521.825111] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1521.825277] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1521.825611] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1521.825778] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1521.825962] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1521.826195] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1521.826405] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1521.827308] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0348d711-e669-4974-b608-fdb5a1619bdf {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1521.837145] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4728b0d0-7865-4330-ae2a-c038bdd37066 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1522.328790] env[67752]: DEBUG nova.network.neutron [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Successfully created port: 9f8abe77-eced-4692-953f-774ba8d381db {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1522.928432] env[67752]: DEBUG nova.network.neutron [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Successfully updated port: 9f8abe77-eced-4692-953f-774ba8d381db {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1522.943221] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "refresh_cache-0f3f8622-4b01-47d3-bbbb-dcd88c7ca010" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1522.943447] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquired lock "refresh_cache-0f3f8622-4b01-47d3-bbbb-dcd88c7ca010" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1522.943688] env[67752]: DEBUG nova.network.neutron [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1522.998469] env[67752]: DEBUG nova.network.neutron [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1523.269775] env[67752]: DEBUG nova.network.neutron [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Updating instance_info_cache with network_info: [{"id": "9f8abe77-eced-4692-953f-774ba8d381db", "address": "fa:16:3e:9c:cd:ff", "network": {"id": "b761e2cd-93bf-4743-989c-8580aaf2f67a", "bridge": "br-int", "label": "tempest-MultipleCreateTestJSON-1617981042-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d6b1ff6c8aaa49bd968d23a1fe068529", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "e85cbc56-fee0-41f7-bc70-64f31775ce92", "external-id": "nsx-vlan-transportzone-793", "segmentation_id": 793, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9f8abe77-ec", "ovs_interfaceid": "9f8abe77-eced-4692-953f-774ba8d381db", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1523.284319] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Releasing lock "refresh_cache-0f3f8622-4b01-47d3-bbbb-dcd88c7ca010" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1523.284807] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Instance network_info: |[{"id": "9f8abe77-eced-4692-953f-774ba8d381db", "address": "fa:16:3e:9c:cd:ff", "network": {"id": "b761e2cd-93bf-4743-989c-8580aaf2f67a", "bridge": "br-int", "label": "tempest-MultipleCreateTestJSON-1617981042-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d6b1ff6c8aaa49bd968d23a1fe068529", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "e85cbc56-fee0-41f7-bc70-64f31775ce92", "external-id": "nsx-vlan-transportzone-793", "segmentation_id": 793, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9f8abe77-ec", "ovs_interfaceid": "9f8abe77-eced-4692-953f-774ba8d381db", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1523.285580] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:9c:cd:ff', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'e85cbc56-fee0-41f7-bc70-64f31775ce92', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '9f8abe77-eced-4692-953f-774ba8d381db', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1523.293909] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Creating folder: Project (d6b1ff6c8aaa49bd968d23a1fe068529). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1523.294851] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-607ef3ea-b80c-4518-bb2b-a217a5230c37 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1523.305727] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Created folder: Project (d6b1ff6c8aaa49bd968d23a1fe068529) in parent group-v639722. [ 1523.305963] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Creating folder: Instances. Parent ref: group-v639821. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1523.306248] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-93c0efed-66f7-46a1-af9d-8cba5e847343 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1523.314853] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Created folder: Instances in parent group-v639821. [ 1523.315129] env[67752]: DEBUG oslo.service.loopingcall [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1523.315315] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1523.315566] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-e9f0eb1b-0496-47b5-95d4-9430e26da5af {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1523.335339] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1523.335339] env[67752]: value = "task-3199793" [ 1523.335339] env[67752]: _type = "Task" [ 1523.335339] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1523.342177] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199793, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1523.383414] env[67752]: DEBUG nova.compute.manager [req-1a9d6ef6-93e1-4080-99bb-a0c69056ca14 req-f5f92a52-1b9e-4393-84b1-fa4e9602e363 service nova] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Received event network-vif-plugged-9f8abe77-eced-4692-953f-774ba8d381db {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1523.384023] env[67752]: DEBUG oslo_concurrency.lockutils [req-1a9d6ef6-93e1-4080-99bb-a0c69056ca14 req-f5f92a52-1b9e-4393-84b1-fa4e9602e363 service nova] Acquiring lock "0f3f8622-4b01-47d3-bbbb-dcd88c7ca010-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1523.385083] env[67752]: DEBUG oslo_concurrency.lockutils [req-1a9d6ef6-93e1-4080-99bb-a0c69056ca14 req-f5f92a52-1b9e-4393-84b1-fa4e9602e363 service nova] Lock "0f3f8622-4b01-47d3-bbbb-dcd88c7ca010-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1523.385083] env[67752]: DEBUG oslo_concurrency.lockutils [req-1a9d6ef6-93e1-4080-99bb-a0c69056ca14 req-f5f92a52-1b9e-4393-84b1-fa4e9602e363 service nova] Lock "0f3f8622-4b01-47d3-bbbb-dcd88c7ca010-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1523.385083] env[67752]: DEBUG nova.compute.manager [req-1a9d6ef6-93e1-4080-99bb-a0c69056ca14 req-f5f92a52-1b9e-4393-84b1-fa4e9602e363 service nova] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] No waiting events found dispatching network-vif-plugged-9f8abe77-eced-4692-953f-774ba8d381db {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1523.385083] env[67752]: WARNING nova.compute.manager [req-1a9d6ef6-93e1-4080-99bb-a0c69056ca14 req-f5f92a52-1b9e-4393-84b1-fa4e9602e363 service nova] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Received unexpected event network-vif-plugged-9f8abe77-eced-4692-953f-774ba8d381db for instance with vm_state building and task_state spawning. [ 1523.385523] env[67752]: DEBUG nova.compute.manager [req-1a9d6ef6-93e1-4080-99bb-a0c69056ca14 req-f5f92a52-1b9e-4393-84b1-fa4e9602e363 service nova] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Received event network-changed-9f8abe77-eced-4692-953f-774ba8d381db {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1523.386511] env[67752]: DEBUG nova.compute.manager [req-1a9d6ef6-93e1-4080-99bb-a0c69056ca14 req-f5f92a52-1b9e-4393-84b1-fa4e9602e363 service nova] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Refreshing instance network info cache due to event network-changed-9f8abe77-eced-4692-953f-774ba8d381db. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1523.386511] env[67752]: DEBUG oslo_concurrency.lockutils [req-1a9d6ef6-93e1-4080-99bb-a0c69056ca14 req-f5f92a52-1b9e-4393-84b1-fa4e9602e363 service nova] Acquiring lock "refresh_cache-0f3f8622-4b01-47d3-bbbb-dcd88c7ca010" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1523.386511] env[67752]: DEBUG oslo_concurrency.lockutils [req-1a9d6ef6-93e1-4080-99bb-a0c69056ca14 req-f5f92a52-1b9e-4393-84b1-fa4e9602e363 service nova] Acquired lock "refresh_cache-0f3f8622-4b01-47d3-bbbb-dcd88c7ca010" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1523.386779] env[67752]: DEBUG nova.network.neutron [req-1a9d6ef6-93e1-4080-99bb-a0c69056ca14 req-f5f92a52-1b9e-4393-84b1-fa4e9602e363 service nova] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Refreshing network info cache for port 9f8abe77-eced-4692-953f-774ba8d381db {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1523.798821] env[67752]: DEBUG nova.network.neutron [req-1a9d6ef6-93e1-4080-99bb-a0c69056ca14 req-f5f92a52-1b9e-4393-84b1-fa4e9602e363 service nova] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Updated VIF entry in instance network info cache for port 9f8abe77-eced-4692-953f-774ba8d381db. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1523.799191] env[67752]: DEBUG nova.network.neutron [req-1a9d6ef6-93e1-4080-99bb-a0c69056ca14 req-f5f92a52-1b9e-4393-84b1-fa4e9602e363 service nova] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Updating instance_info_cache with network_info: [{"id": "9f8abe77-eced-4692-953f-774ba8d381db", "address": "fa:16:3e:9c:cd:ff", "network": {"id": "b761e2cd-93bf-4743-989c-8580aaf2f67a", "bridge": "br-int", "label": "tempest-MultipleCreateTestJSON-1617981042-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.4", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d6b1ff6c8aaa49bd968d23a1fe068529", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "e85cbc56-fee0-41f7-bc70-64f31775ce92", "external-id": "nsx-vlan-transportzone-793", "segmentation_id": 793, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap9f8abe77-ec", "ovs_interfaceid": "9f8abe77-eced-4692-953f-774ba8d381db", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1523.809407] env[67752]: DEBUG oslo_concurrency.lockutils [req-1a9d6ef6-93e1-4080-99bb-a0c69056ca14 req-f5f92a52-1b9e-4393-84b1-fa4e9602e363 service nova] Releasing lock "refresh_cache-0f3f8622-4b01-47d3-bbbb-dcd88c7ca010" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1523.844062] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199793, 'name': CreateVM_Task, 'duration_secs': 0.282969} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1523.844224] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1523.844865] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1523.845040] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1523.845356] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1523.845591] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b9df47d1-cfbb-46c5-ba61-3a311081ed16 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1523.849930] env[67752]: DEBUG oslo_vmware.api [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for the task: (returnval){ [ 1523.849930] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52131970-d7e3-26d8-1ce8-45d6bd25eff0" [ 1523.849930] env[67752]: _type = "Task" [ 1523.849930] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1523.857350] env[67752]: DEBUG oslo_vmware.api [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52131970-d7e3-26d8-1ce8-45d6bd25eff0, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1524.361142] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1524.361471] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1524.361609] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1538.798079] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bcfd9a26-e4d2-4ae9-a941-1ca1dc5e30d4 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquiring lock "c1975c17-15a5-4751-bf7f-d5b7e0d2fea1" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1562.107829] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1562.634677] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1567.635044] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1567.635431] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 1567.635431] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 1567.658222] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1567.658373] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: d3940c29-852a-427b-9027-aa4080150724] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1567.658510] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1567.658640] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1567.658782] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1567.658912] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1567.659085] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1567.659221] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1567.659342] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1567.659461] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1567.659580] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 1567.660059] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1567.660243] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1567.660397] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1567.660526] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 1568.358322] env[67752]: WARNING oslo_vmware.rw_handles [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1568.358322] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1568.358322] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1568.358322] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1568.358322] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1568.358322] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1568.358322] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1568.358322] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1568.358322] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1568.358322] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1568.358322] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1568.358322] env[67752]: ERROR oslo_vmware.rw_handles [ 1568.358701] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/e922adc0-f8d9-4116-8063-c8180f526013/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1568.360943] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1568.361242] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Copying Virtual Disk [datastore2] vmware_temp/e922adc0-f8d9-4116-8063-c8180f526013/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/e922adc0-f8d9-4116-8063-c8180f526013/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1568.361529] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-6245d5af-e84b-4def-8243-eebf215ed179 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1568.369285] env[67752]: DEBUG oslo_vmware.api [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Waiting for the task: (returnval){ [ 1568.369285] env[67752]: value = "task-3199794" [ 1568.369285] env[67752]: _type = "Task" [ 1568.369285] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1568.377575] env[67752]: DEBUG oslo_vmware.api [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Task: {'id': task-3199794, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1568.635568] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1568.880076] env[67752]: DEBUG oslo_vmware.exceptions [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1568.880342] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1568.880879] env[67752]: ERROR nova.compute.manager [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1568.880879] env[67752]: Faults: ['InvalidArgument'] [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Traceback (most recent call last): [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] yield resources [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] self.driver.spawn(context, instance, image_meta, [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] self._fetch_image_if_missing(context, vi) [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] image_cache(vi, tmp_image_ds_loc) [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] vm_util.copy_virtual_disk( [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] session._wait_for_task(vmdk_copy_task) [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] return self.wait_for_task(task_ref) [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] return evt.wait() [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] result = hub.switch() [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] return self.greenlet.switch() [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] self.f(*self.args, **self.kw) [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] raise exceptions.translate_fault(task_info.error) [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Faults: ['InvalidArgument'] [ 1568.880879] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] [ 1568.881959] env[67752]: INFO nova.compute.manager [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Terminating instance [ 1568.882752] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1568.882954] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1568.883203] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-c49575aa-20f2-4c42-8fdc-148c4d081473 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1568.885330] env[67752]: DEBUG nova.compute.manager [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1568.885526] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1568.886287] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0fa0eba8-492d-41bb-baf4-06527584429f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1568.892919] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1568.893142] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-0bcfab42-a1ab-43bc-ad1a-abf0db07819c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1568.895171] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1568.895358] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1568.896293] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-4e510d02-9827-40a0-82c2-3dcfaaeeafc9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1568.901032] env[67752]: DEBUG oslo_vmware.api [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Waiting for the task: (returnval){ [ 1568.901032] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52b41597-e01f-8ab8-c0d4-410b02621bd0" [ 1568.901032] env[67752]: _type = "Task" [ 1568.901032] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1568.910967] env[67752]: DEBUG oslo_vmware.api [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52b41597-e01f-8ab8-c0d4-410b02621bd0, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1568.960026] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1568.960283] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1568.960471] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Deleting the datastore file [datastore2] f6499ee5-cb24-44e1-9fe7-c19feeb732eb {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1568.960742] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-172e9ad3-f6a1-433a-aff1-d36c862dd354 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1568.966508] env[67752]: DEBUG oslo_vmware.api [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Waiting for the task: (returnval){ [ 1568.966508] env[67752]: value = "task-3199796" [ 1568.966508] env[67752]: _type = "Task" [ 1568.966508] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1568.974350] env[67752]: DEBUG oslo_vmware.api [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Task: {'id': task-3199796, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1569.411231] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1569.411452] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Creating directory with path [datastore2] vmware_temp/ce6fc038-ed22-46f6-9934-26f4efc41e99/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1569.411680] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-d3e14d70-8012-473f-9e5f-5bafd0a048b9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1569.424071] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Created directory with path [datastore2] vmware_temp/ce6fc038-ed22-46f6-9934-26f4efc41e99/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1569.424272] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Fetch image to [datastore2] vmware_temp/ce6fc038-ed22-46f6-9934-26f4efc41e99/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1569.424445] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/ce6fc038-ed22-46f6-9934-26f4efc41e99/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1569.425309] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1ffb0d6b-9e12-460e-92ca-5c4da594b0ce {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1569.432144] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e50f38a3-0244-4d02-83af-509740e7b36d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1569.441252] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c51d745d-0e3b-4006-b31e-299f724259ef {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1569.475703] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee8dbba6-f9a3-475d-ac98-d44254ac155f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1569.485641] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-0fa09ea4-d900-4c29-abc9-583d220a3a8e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1569.487316] env[67752]: DEBUG oslo_vmware.api [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Task: {'id': task-3199796, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.084561} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1569.487552] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1569.487735] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1569.487908] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1569.488096] env[67752]: INFO nova.compute.manager [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1569.490559] env[67752]: DEBUG nova.compute.claims [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1569.490729] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1569.490949] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1569.512701] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1569.590134] env[67752]: DEBUG oslo_vmware.rw_handles [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/ce6fc038-ed22-46f6-9934-26f4efc41e99/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1569.652987] env[67752]: DEBUG oslo_vmware.rw_handles [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1569.653366] env[67752]: DEBUG oslo_vmware.rw_handles [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/ce6fc038-ed22-46f6-9934-26f4efc41e99/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1569.778963] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-acbe08f1-2644-41fe-ac6d-b50b460bc94a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1569.786731] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-789386c4-ede0-4faa-8546-e71ed6606186 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1569.815356] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5bd3f76f-5e8f-446f-982d-7dbc6882f5e1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1569.822193] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ed10f2a-56f3-43c9-aec6-28c2a3c79bb6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1569.835837] env[67752]: DEBUG nova.compute.provider_tree [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1569.845937] env[67752]: DEBUG nova.scheduler.client.report [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1569.859876] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.369s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1569.860441] env[67752]: ERROR nova.compute.manager [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1569.860441] env[67752]: Faults: ['InvalidArgument'] [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Traceback (most recent call last): [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] self.driver.spawn(context, instance, image_meta, [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] self._fetch_image_if_missing(context, vi) [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] image_cache(vi, tmp_image_ds_loc) [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] vm_util.copy_virtual_disk( [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] session._wait_for_task(vmdk_copy_task) [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] return self.wait_for_task(task_ref) [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] return evt.wait() [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] result = hub.switch() [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] return self.greenlet.switch() [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] self.f(*self.args, **self.kw) [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] raise exceptions.translate_fault(task_info.error) [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Faults: ['InvalidArgument'] [ 1569.860441] env[67752]: ERROR nova.compute.manager [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] [ 1569.861301] env[67752]: DEBUG nova.compute.utils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1569.862852] env[67752]: DEBUG nova.compute.manager [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Build of instance f6499ee5-cb24-44e1-9fe7-c19feeb732eb was re-scheduled: A specified parameter was not correct: fileType [ 1569.862852] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1569.863266] env[67752]: DEBUG nova.compute.manager [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1569.863443] env[67752]: DEBUG nova.compute.manager [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1569.863616] env[67752]: DEBUG nova.compute.manager [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1569.863779] env[67752]: DEBUG nova.network.neutron [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1570.174030] env[67752]: DEBUG nova.network.neutron [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1570.186109] env[67752]: INFO nova.compute.manager [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Took 0.32 seconds to deallocate network for instance. [ 1570.279555] env[67752]: INFO nova.scheduler.client.report [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Deleted allocations for instance f6499ee5-cb24-44e1-9fe7-c19feeb732eb [ 1570.303418] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7c3113d1-70a0-4cc0-a615-18f1fea5258d tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "f6499ee5-cb24-44e1-9fe7-c19feeb732eb" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 634.558s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1570.304600] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b2e6be7b-51f6-480d-b71c-50c2efb458f3 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "f6499ee5-cb24-44e1-9fe7-c19feeb732eb" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 436.970s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1570.304877] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b2e6be7b-51f6-480d-b71c-50c2efb458f3 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Acquiring lock "f6499ee5-cb24-44e1-9fe7-c19feeb732eb-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1570.305121] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b2e6be7b-51f6-480d-b71c-50c2efb458f3 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "f6499ee5-cb24-44e1-9fe7-c19feeb732eb-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1570.305297] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b2e6be7b-51f6-480d-b71c-50c2efb458f3 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "f6499ee5-cb24-44e1-9fe7-c19feeb732eb-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1570.308313] env[67752]: INFO nova.compute.manager [None req-b2e6be7b-51f6-480d-b71c-50c2efb458f3 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Terminating instance [ 1570.309809] env[67752]: DEBUG nova.compute.manager [None req-b2e6be7b-51f6-480d-b71c-50c2efb458f3 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1570.310030] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-b2e6be7b-51f6-480d-b71c-50c2efb458f3 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1570.310463] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b7393527-1188-40f4-bef8-180fa450c813 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1570.315101] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1570.321407] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c46135b4-6209-4553-a522-db8389d3e38b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1570.350739] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-b2e6be7b-51f6-480d-b71c-50c2efb458f3 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance f6499ee5-cb24-44e1-9fe7-c19feeb732eb could not be found. [ 1570.350958] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-b2e6be7b-51f6-480d-b71c-50c2efb458f3 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1570.351184] env[67752]: INFO nova.compute.manager [None req-b2e6be7b-51f6-480d-b71c-50c2efb458f3 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1570.351437] env[67752]: DEBUG oslo.service.loopingcall [None req-b2e6be7b-51f6-480d-b71c-50c2efb458f3 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1570.356018] env[67752]: DEBUG nova.compute.manager [-] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1570.356131] env[67752]: DEBUG nova.network.neutron [-] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1570.367996] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1570.368272] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1570.369708] env[67752]: INFO nova.compute.claims [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1570.393312] env[67752]: DEBUG nova.network.neutron [-] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1570.411270] env[67752]: INFO nova.compute.manager [-] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] Took 0.06 seconds to deallocate network for instance. [ 1570.510951] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b2e6be7b-51f6-480d-b71c-50c2efb458f3 tempest-VolumesAdminNegativeTest-89224081 tempest-VolumesAdminNegativeTest-89224081-project-member] Lock "f6499ee5-cb24-44e1-9fe7-c19feeb732eb" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.206s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1570.511971] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "f6499ee5-cb24-44e1-9fe7-c19feeb732eb" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 390.870s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1570.512209] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: f6499ee5-cb24-44e1-9fe7-c19feeb732eb] During sync_power_state the instance has a pending task (deleting). Skip. [ 1570.512411] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "f6499ee5-cb24-44e1-9fe7-c19feeb732eb" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1570.627024] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bfaee1ef-70a2-4dbf-9b79-46647f4ab0da {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1570.634780] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e092abec-54be-4c30-b0ae-8a46a7cc567d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1570.666537] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2d129e1-3014-4f57-8e35-7bae70c54f4a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1570.674036] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f6483d53-242e-424e-956b-ddcd9f75f014 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1570.686879] env[67752]: DEBUG nova.compute.provider_tree [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1570.698023] env[67752]: DEBUG nova.scheduler.client.report [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1570.715592] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.347s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1570.716439] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1570.757022] env[67752]: DEBUG nova.compute.utils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1570.757022] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1570.757022] env[67752]: DEBUG nova.network.neutron [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1570.767825] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1570.812454] env[67752]: DEBUG nova.policy [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '827cdef2d9b647f79e05f8d23696a090', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd6b1ff6c8aaa49bd968d23a1fe068529', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1570.829669] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1570.855039] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1570.855298] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1570.855456] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1570.855636] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1570.855860] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1570.856032] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1570.856247] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1570.856408] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1570.856575] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1570.856740] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1570.856920] env[67752]: DEBUG nova.virt.hardware [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1570.857765] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1fa1e1eb-b203-4ae4-8d82-59dd64644be9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1570.865580] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-419a6728-0615-4b9d-9461-ea32e57ebc8d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1571.107379] env[67752]: DEBUG nova.network.neutron [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Successfully created port: 6488d828-9f4e-4ad2-a42b-d00fc0801079 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1572.026999] env[67752]: DEBUG nova.network.neutron [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Successfully updated port: 6488d828-9f4e-4ad2-a42b-d00fc0801079 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1572.037366] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "refresh_cache-7557e3a1-b298-401f-8320-5a48ddda3298" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1572.037504] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquired lock "refresh_cache-7557e3a1-b298-401f-8320-5a48ddda3298" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1572.037654] env[67752]: DEBUG nova.network.neutron [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1572.077028] env[67752]: DEBUG nova.network.neutron [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1572.232103] env[67752]: DEBUG nova.compute.manager [req-639d9755-3bed-4341-af38-9f4f15cd5d0d req-c6f67430-93d9-4c72-bfee-0b5fa2a12a6e service nova] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Received event network-vif-plugged-6488d828-9f4e-4ad2-a42b-d00fc0801079 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1572.232352] env[67752]: DEBUG oslo_concurrency.lockutils [req-639d9755-3bed-4341-af38-9f4f15cd5d0d req-c6f67430-93d9-4c72-bfee-0b5fa2a12a6e service nova] Acquiring lock "7557e3a1-b298-401f-8320-5a48ddda3298-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1572.232997] env[67752]: DEBUG oslo_concurrency.lockutils [req-639d9755-3bed-4341-af38-9f4f15cd5d0d req-c6f67430-93d9-4c72-bfee-0b5fa2a12a6e service nova] Lock "7557e3a1-b298-401f-8320-5a48ddda3298-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1572.233266] env[67752]: DEBUG oslo_concurrency.lockutils [req-639d9755-3bed-4341-af38-9f4f15cd5d0d req-c6f67430-93d9-4c72-bfee-0b5fa2a12a6e service nova] Lock "7557e3a1-b298-401f-8320-5a48ddda3298-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1572.233458] env[67752]: DEBUG nova.compute.manager [req-639d9755-3bed-4341-af38-9f4f15cd5d0d req-c6f67430-93d9-4c72-bfee-0b5fa2a12a6e service nova] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] No waiting events found dispatching network-vif-plugged-6488d828-9f4e-4ad2-a42b-d00fc0801079 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1572.233633] env[67752]: WARNING nova.compute.manager [req-639d9755-3bed-4341-af38-9f4f15cd5d0d req-c6f67430-93d9-4c72-bfee-0b5fa2a12a6e service nova] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Received unexpected event network-vif-plugged-6488d828-9f4e-4ad2-a42b-d00fc0801079 for instance with vm_state building and task_state spawning. [ 1572.233793] env[67752]: DEBUG nova.compute.manager [req-639d9755-3bed-4341-af38-9f4f15cd5d0d req-c6f67430-93d9-4c72-bfee-0b5fa2a12a6e service nova] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Received event network-changed-6488d828-9f4e-4ad2-a42b-d00fc0801079 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1572.233949] env[67752]: DEBUG nova.compute.manager [req-639d9755-3bed-4341-af38-9f4f15cd5d0d req-c6f67430-93d9-4c72-bfee-0b5fa2a12a6e service nova] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Refreshing instance network info cache due to event network-changed-6488d828-9f4e-4ad2-a42b-d00fc0801079. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1572.234138] env[67752]: DEBUG oslo_concurrency.lockutils [req-639d9755-3bed-4341-af38-9f4f15cd5d0d req-c6f67430-93d9-4c72-bfee-0b5fa2a12a6e service nova] Acquiring lock "refresh_cache-7557e3a1-b298-401f-8320-5a48ddda3298" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1572.237575] env[67752]: DEBUG nova.network.neutron [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Updating instance_info_cache with network_info: [{"id": "6488d828-9f4e-4ad2-a42b-d00fc0801079", "address": "fa:16:3e:4d:af:94", "network": {"id": "b761e2cd-93bf-4743-989c-8580aaf2f67a", "bridge": "br-int", "label": "tempest-MultipleCreateTestJSON-1617981042-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d6b1ff6c8aaa49bd968d23a1fe068529", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "e85cbc56-fee0-41f7-bc70-64f31775ce92", "external-id": "nsx-vlan-transportzone-793", "segmentation_id": 793, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap6488d828-9f", "ovs_interfaceid": "6488d828-9f4e-4ad2-a42b-d00fc0801079", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1572.248191] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Releasing lock "refresh_cache-7557e3a1-b298-401f-8320-5a48ddda3298" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1572.248460] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Instance network_info: |[{"id": "6488d828-9f4e-4ad2-a42b-d00fc0801079", "address": "fa:16:3e:4d:af:94", "network": {"id": "b761e2cd-93bf-4743-989c-8580aaf2f67a", "bridge": "br-int", "label": "tempest-MultipleCreateTestJSON-1617981042-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d6b1ff6c8aaa49bd968d23a1fe068529", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "e85cbc56-fee0-41f7-bc70-64f31775ce92", "external-id": "nsx-vlan-transportzone-793", "segmentation_id": 793, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap6488d828-9f", "ovs_interfaceid": "6488d828-9f4e-4ad2-a42b-d00fc0801079", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1572.248738] env[67752]: DEBUG oslo_concurrency.lockutils [req-639d9755-3bed-4341-af38-9f4f15cd5d0d req-c6f67430-93d9-4c72-bfee-0b5fa2a12a6e service nova] Acquired lock "refresh_cache-7557e3a1-b298-401f-8320-5a48ddda3298" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1572.248917] env[67752]: DEBUG nova.network.neutron [req-639d9755-3bed-4341-af38-9f4f15cd5d0d req-c6f67430-93d9-4c72-bfee-0b5fa2a12a6e service nova] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Refreshing network info cache for port 6488d828-9f4e-4ad2-a42b-d00fc0801079 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1572.250043] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:4d:af:94', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'e85cbc56-fee0-41f7-bc70-64f31775ce92', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '6488d828-9f4e-4ad2-a42b-d00fc0801079', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1572.257379] env[67752]: DEBUG oslo.service.loopingcall [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1572.258334] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1572.260531] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-b81653e2-a0b5-4bcd-af73-c352c1b0df47 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1572.280552] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1572.280552] env[67752]: value = "task-3199797" [ 1572.280552] env[67752]: _type = "Task" [ 1572.280552] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1572.288527] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199797, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1572.514966] env[67752]: DEBUG nova.network.neutron [req-639d9755-3bed-4341-af38-9f4f15cd5d0d req-c6f67430-93d9-4c72-bfee-0b5fa2a12a6e service nova] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Updated VIF entry in instance network info cache for port 6488d828-9f4e-4ad2-a42b-d00fc0801079. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1572.515421] env[67752]: DEBUG nova.network.neutron [req-639d9755-3bed-4341-af38-9f4f15cd5d0d req-c6f67430-93d9-4c72-bfee-0b5fa2a12a6e service nova] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Updating instance_info_cache with network_info: [{"id": "6488d828-9f4e-4ad2-a42b-d00fc0801079", "address": "fa:16:3e:4d:af:94", "network": {"id": "b761e2cd-93bf-4743-989c-8580aaf2f67a", "bridge": "br-int", "label": "tempest-MultipleCreateTestJSON-1617981042-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d6b1ff6c8aaa49bd968d23a1fe068529", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "e85cbc56-fee0-41f7-bc70-64f31775ce92", "external-id": "nsx-vlan-transportzone-793", "segmentation_id": 793, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap6488d828-9f", "ovs_interfaceid": "6488d828-9f4e-4ad2-a42b-d00fc0801079", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1572.524808] env[67752]: DEBUG oslo_concurrency.lockutils [req-639d9755-3bed-4341-af38-9f4f15cd5d0d req-c6f67430-93d9-4c72-bfee-0b5fa2a12a6e service nova] Releasing lock "refresh_cache-7557e3a1-b298-401f-8320-5a48ddda3298" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1572.791356] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199797, 'name': CreateVM_Task, 'duration_secs': 0.285159} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1572.791534] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1572.792230] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1572.792406] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1572.792745] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1572.792993] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-f637e88d-f205-4db7-91f6-ee371d61ff09 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1572.797203] env[67752]: DEBUG oslo_vmware.api [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for the task: (returnval){ [ 1572.797203] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]528e9ab1-f2f0-6da5-a2b1-410cf7751a09" [ 1572.797203] env[67752]: _type = "Task" [ 1572.797203] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1572.804558] env[67752]: DEBUG oslo_vmware.api [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]528e9ab1-f2f0-6da5-a2b1-410cf7751a09, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1573.307402] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1573.307702] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1573.307798] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1573.634214] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1573.634503] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1573.646009] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1573.646210] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1573.646386] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1573.646539] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1573.647605] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3aa947b3-f69d-45b2-97b7-cdbe067d7184 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1573.655585] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c1ca4e35-53c8-4c59-a521-aee93e29adde {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1573.668736] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5250c0ed-72e7-4a4b-8224-ed4d217efd8c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1573.674484] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa9de2dc-e042-4bd2-a2eb-be02163a74ae {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1573.703749] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180982MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1573.703898] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1573.704110] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1573.776011] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d3940c29-852a-427b-9027-aa4080150724 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1573.776195] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1573.776326] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance a8bd310e-530a-46ab-add7-1b827ea5f399 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1573.776451] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9432cd04-c2df-428b-b31a-93d3ec0c9d1f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1573.776570] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 90e58033-eacd-433a-8d25-40f7d8403588 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1573.776687] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 81738fe6-6d50-4dfe-ae86-4c8771837168 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1573.776840] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e4ebb0f0-ab58-491f-bc4a-7548bbe415d8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1573.776978] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c1975c17-15a5-4751-bf7f-d5b7e0d2fea1 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1573.777110] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1573.777490] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 7557e3a1-b298-401f-8320-5a48ddda3298 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1573.788148] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 34c75d7e-e63e-4eff-afad-0edd61e023fc has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1573.798126] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1227bf67-5435-4c53-86ac-224e7e00d86c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1573.810858] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 214ac8b8-c04c-47cf-8dfd-fba169d6ec44 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1573.821811] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 7ab65bf3-7bbd-4c70-8a6d-18691bba92b5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1573.832033] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 048833ec-a7b2-4a8b-9204-788f2e9c5be7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1573.832284] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1573.832447] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1574.012698] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aae4e3a3-1d92-4fa4-a37d-339fb495e224 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1574.020506] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b8f03816-0fb0-4c2e-8ce2-0c1ab6b2a2c4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1574.050721] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ad22fc1-8a03-4673-a2bf-dad13ac0bc2b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1574.058275] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ae595cc-bc1b-4fa3-a1b8-ea8148cc3197 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1574.070813] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1574.079667] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1574.095532] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1574.095713] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.392s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1606.907992] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fd57df97-ba6f-4c42-bc2b-3195c689e06c tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "7557e3a1-b298-401f-8320-5a48ddda3298" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1606.969932] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3b6fbf87-5b81-42cc-aff0-6ea9d5ce826d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "0f3f8622-4b01-47d3-bbbb-dcd88c7ca010" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1615.009054] env[67752]: WARNING oslo_vmware.rw_handles [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1615.009054] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1615.009054] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1615.009054] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1615.009054] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1615.009054] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1615.009054] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1615.009054] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1615.009054] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1615.009054] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1615.009054] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1615.009054] env[67752]: ERROR oslo_vmware.rw_handles [ 1615.009715] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/ce6fc038-ed22-46f6-9934-26f4efc41e99/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1615.011593] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1615.011840] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Copying Virtual Disk [datastore2] vmware_temp/ce6fc038-ed22-46f6-9934-26f4efc41e99/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/ce6fc038-ed22-46f6-9934-26f4efc41e99/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1615.012656] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-aec1ba55-b9f1-47e7-870c-b98f864667c0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1615.020997] env[67752]: DEBUG oslo_vmware.api [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Waiting for the task: (returnval){ [ 1615.020997] env[67752]: value = "task-3199798" [ 1615.020997] env[67752]: _type = "Task" [ 1615.020997] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1615.029200] env[67752]: DEBUG oslo_vmware.api [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Task: {'id': task-3199798, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1615.531355] env[67752]: DEBUG oslo_vmware.exceptions [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1615.531759] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1615.532420] env[67752]: ERROR nova.compute.manager [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1615.532420] env[67752]: Faults: ['InvalidArgument'] [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] Traceback (most recent call last): [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] yield resources [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] self.driver.spawn(context, instance, image_meta, [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] self._fetch_image_if_missing(context, vi) [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] image_cache(vi, tmp_image_ds_loc) [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] vm_util.copy_virtual_disk( [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] session._wait_for_task(vmdk_copy_task) [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] return self.wait_for_task(task_ref) [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] return evt.wait() [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] result = hub.switch() [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] return self.greenlet.switch() [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] self.f(*self.args, **self.kw) [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] raise exceptions.translate_fault(task_info.error) [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] Faults: ['InvalidArgument'] [ 1615.532420] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] [ 1615.533271] env[67752]: INFO nova.compute.manager [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Terminating instance [ 1615.534381] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1615.534588] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1615.534825] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-7e725ebf-1e07-4318-bf6a-9c6c7096aa72 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1615.537053] env[67752]: DEBUG nova.compute.manager [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1615.537251] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1615.537986] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44dc9eac-108f-4756-b094-997a212d7fcc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1615.548018] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1615.548018] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-c20cff4b-4617-4872-abce-20c2131b5965 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1615.548775] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1615.548956] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1615.549915] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-550c81c0-1a5b-406a-b57d-dece769bc026 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1615.555029] env[67752]: DEBUG oslo_vmware.api [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Waiting for the task: (returnval){ [ 1615.555029] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52e31f36-42a1-4270-0d9e-8b247e73bc64" [ 1615.555029] env[67752]: _type = "Task" [ 1615.555029] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1615.566879] env[67752]: DEBUG oslo_vmware.api [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52e31f36-42a1-4270-0d9e-8b247e73bc64, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1615.620506] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1615.620740] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1615.620926] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Deleting the datastore file [datastore2] d3940c29-852a-427b-9027-aa4080150724 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1615.621219] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-705e1e5c-808c-4e3f-a3ab-61c7f6684bef {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1615.627553] env[67752]: DEBUG oslo_vmware.api [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Waiting for the task: (returnval){ [ 1615.627553] env[67752]: value = "task-3199800" [ 1615.627553] env[67752]: _type = "Task" [ 1615.627553] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1615.635501] env[67752]: DEBUG oslo_vmware.api [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Task: {'id': task-3199800, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1616.066047] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1616.066047] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Creating directory with path [datastore2] vmware_temp/04ccb8ff-deab-4fe9-b788-97dad60e8ef5/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1616.066047] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-d37af08a-6241-4617-b85d-70accd118f2c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1616.076798] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Created directory with path [datastore2] vmware_temp/04ccb8ff-deab-4fe9-b788-97dad60e8ef5/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1616.076984] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Fetch image to [datastore2] vmware_temp/04ccb8ff-deab-4fe9-b788-97dad60e8ef5/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1616.077175] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/04ccb8ff-deab-4fe9-b788-97dad60e8ef5/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1616.077884] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b395c2f5-4280-4e4b-98a1-f37ea3ba8b09 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1616.084239] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21901d5e-8b5c-4ca5-9af1-95b036f72dd6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1616.093011] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-123f29af-7393-473f-b86f-ab90eb54c698 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1616.122862] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-00b44f02-dc8c-4ac3-8f40-d1d2f6ef76fb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1616.128098] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-2172308f-d480-452c-8fd4-894eaf8bae54 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1616.136438] env[67752]: DEBUG oslo_vmware.api [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Task: {'id': task-3199800, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.070596} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1616.136665] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1616.136861] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1616.137064] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1616.137244] env[67752]: INFO nova.compute.manager [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1616.139376] env[67752]: DEBUG nova.compute.claims [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1616.139547] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1616.139756] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1616.153909] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1616.201725] env[67752]: DEBUG nova.scheduler.client.report [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Refreshing inventories for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:804}} [ 1616.205326] env[67752]: DEBUG oslo_vmware.rw_handles [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/04ccb8ff-deab-4fe9-b788-97dad60e8ef5/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1616.264581] env[67752]: DEBUG nova.scheduler.client.report [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Updating ProviderTree inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:768}} [ 1616.264811] env[67752]: DEBUG nova.compute.provider_tree [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1616.269010] env[67752]: DEBUG oslo_vmware.rw_handles [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1616.269010] env[67752]: DEBUG oslo_vmware.rw_handles [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/04ccb8ff-deab-4fe9-b788-97dad60e8ef5/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1616.279553] env[67752]: DEBUG nova.scheduler.client.report [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Refreshing aggregate associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, aggregates: None {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:813}} [ 1616.298480] env[67752]: DEBUG nova.scheduler.client.report [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Refreshing trait associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, traits: COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:825}} [ 1616.474257] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3257bacf-c51b-4cee-8e9c-abe97b1ad896 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1616.481584] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1328fc31-0f14-41e3-ab29-b0887fbba6ea {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1616.511741] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-be840ae4-68da-4824-8feb-d5a31a3fe617 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1616.518178] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b912098b-44ad-4916-81fd-900d1f90412f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1616.530499] env[67752]: DEBUG nova.compute.provider_tree [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1616.540527] env[67752]: DEBUG nova.scheduler.client.report [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1616.554828] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.415s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1616.555359] env[67752]: ERROR nova.compute.manager [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1616.555359] env[67752]: Faults: ['InvalidArgument'] [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] Traceback (most recent call last): [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] self.driver.spawn(context, instance, image_meta, [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] self._fetch_image_if_missing(context, vi) [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] image_cache(vi, tmp_image_ds_loc) [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] vm_util.copy_virtual_disk( [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] session._wait_for_task(vmdk_copy_task) [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] return self.wait_for_task(task_ref) [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] return evt.wait() [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] result = hub.switch() [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] return self.greenlet.switch() [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] self.f(*self.args, **self.kw) [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] raise exceptions.translate_fault(task_info.error) [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] Faults: ['InvalidArgument'] [ 1616.555359] env[67752]: ERROR nova.compute.manager [instance: d3940c29-852a-427b-9027-aa4080150724] [ 1616.556227] env[67752]: DEBUG nova.compute.utils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1616.557451] env[67752]: DEBUG nova.compute.manager [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Build of instance d3940c29-852a-427b-9027-aa4080150724 was re-scheduled: A specified parameter was not correct: fileType [ 1616.557451] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1616.557815] env[67752]: DEBUG nova.compute.manager [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1616.557991] env[67752]: DEBUG nova.compute.manager [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1616.558215] env[67752]: DEBUG nova.compute.manager [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1616.558388] env[67752]: DEBUG nova.network.neutron [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1617.288860] env[67752]: DEBUG nova.network.neutron [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1617.300780] env[67752]: INFO nova.compute.manager [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Took 0.74 seconds to deallocate network for instance. [ 1617.423202] env[67752]: INFO nova.scheduler.client.report [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Deleted allocations for instance d3940c29-852a-427b-9027-aa4080150724 [ 1617.457711] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2a921e55-d71f-4285-b99c-c501fc3a1e86 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Lock "d3940c29-852a-427b-9027-aa4080150724" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 604.432s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1617.462032] env[67752]: DEBUG oslo_concurrency.lockutils [None req-723fed07-ce60-447e-acf7-4365158dbdc1 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Lock "d3940c29-852a-427b-9027-aa4080150724" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 407.623s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1617.462032] env[67752]: DEBUG oslo_concurrency.lockutils [None req-723fed07-ce60-447e-acf7-4365158dbdc1 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Acquiring lock "d3940c29-852a-427b-9027-aa4080150724-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1617.462032] env[67752]: DEBUG oslo_concurrency.lockutils [None req-723fed07-ce60-447e-acf7-4365158dbdc1 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Lock "d3940c29-852a-427b-9027-aa4080150724-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1617.462032] env[67752]: DEBUG oslo_concurrency.lockutils [None req-723fed07-ce60-447e-acf7-4365158dbdc1 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Lock "d3940c29-852a-427b-9027-aa4080150724-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1617.462032] env[67752]: INFO nova.compute.manager [None req-723fed07-ce60-447e-acf7-4365158dbdc1 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Terminating instance [ 1617.466492] env[67752]: DEBUG nova.compute.manager [None req-723fed07-ce60-447e-acf7-4365158dbdc1 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1617.466719] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-723fed07-ce60-447e-acf7-4365158dbdc1 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1617.466973] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-40c17812-c5d2-438f-887b-7290cbf0b70c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1617.470379] env[67752]: DEBUG nova.compute.manager [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1617.478617] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-34463c08-ace8-428f-ad31-a62ea94b27b2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1617.510412] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-723fed07-ce60-447e-acf7-4365158dbdc1 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d3940c29-852a-427b-9027-aa4080150724 could not be found. [ 1617.510656] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-723fed07-ce60-447e-acf7-4365158dbdc1 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1617.510931] env[67752]: INFO nova.compute.manager [None req-723fed07-ce60-447e-acf7-4365158dbdc1 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] [instance: d3940c29-852a-427b-9027-aa4080150724] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1617.511184] env[67752]: DEBUG oslo.service.loopingcall [None req-723fed07-ce60-447e-acf7-4365158dbdc1 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1617.511454] env[67752]: DEBUG nova.compute.manager [-] [instance: d3940c29-852a-427b-9027-aa4080150724] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1617.511553] env[67752]: DEBUG nova.network.neutron [-] [instance: d3940c29-852a-427b-9027-aa4080150724] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1617.541848] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1617.541848] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1617.543422] env[67752]: INFO nova.compute.claims [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1617.550161] env[67752]: DEBUG nova.network.neutron [-] [instance: d3940c29-852a-427b-9027-aa4080150724] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1617.563523] env[67752]: INFO nova.compute.manager [-] [instance: d3940c29-852a-427b-9027-aa4080150724] Took 0.05 seconds to deallocate network for instance. [ 1617.697106] env[67752]: DEBUG oslo_concurrency.lockutils [None req-723fed07-ce60-447e-acf7-4365158dbdc1 tempest-InstanceActionsNegativeTestJSON-1217099375 tempest-InstanceActionsNegativeTestJSON-1217099375-project-member] Lock "d3940c29-852a-427b-9027-aa4080150724" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.238s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1617.818751] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b205e9a0-8d01-4b67-a8e2-b9d2255bdc3e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1617.826466] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-39e17934-2b52-4c28-a5f2-4a772068373c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1617.856249] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d6e632d8-3437-4581-b729-df89c4328c46 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1617.863356] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ef0d3a8-1060-4a28-918e-c2f906098958 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1617.877415] env[67752]: DEBUG nova.compute.provider_tree [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1617.887150] env[67752]: DEBUG nova.scheduler.client.report [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1617.901598] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.360s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1617.902113] env[67752]: DEBUG nova.compute.manager [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1617.936672] env[67752]: DEBUG nova.compute.utils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1617.938219] env[67752]: DEBUG nova.compute.manager [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1617.938392] env[67752]: DEBUG nova.network.neutron [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1617.950275] env[67752]: DEBUG nova.compute.manager [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1618.021156] env[67752]: DEBUG nova.policy [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '641d021b5cfb4a3885a3db7932407a38', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e117ebfe10cb4714b149d6e9bc132273', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1618.025240] env[67752]: DEBUG nova.compute.manager [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1618.052196] env[67752]: DEBUG nova.virt.hardware [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1618.052196] env[67752]: DEBUG nova.virt.hardware [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1618.052363] env[67752]: DEBUG nova.virt.hardware [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1618.052483] env[67752]: DEBUG nova.virt.hardware [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1618.052613] env[67752]: DEBUG nova.virt.hardware [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1618.052770] env[67752]: DEBUG nova.virt.hardware [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1618.053092] env[67752]: DEBUG nova.virt.hardware [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1618.053376] env[67752]: DEBUG nova.virt.hardware [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1618.053574] env[67752]: DEBUG nova.virt.hardware [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1618.053745] env[67752]: DEBUG nova.virt.hardware [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1618.053983] env[67752]: DEBUG nova.virt.hardware [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1618.055019] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-587f3c52-e0bd-4e29-a259-a023e6159658 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1618.064643] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-81516697-f54d-4cb0-b081-db5ec771a550 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1618.505136] env[67752]: DEBUG nova.network.neutron [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Successfully created port: 5682762b-1f55-416f-9dc9-47c9b0bd52c0 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1619.738676] env[67752]: DEBUG nova.network.neutron [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Successfully updated port: 5682762b-1f55-416f-9dc9-47c9b0bd52c0 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1619.750330] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "refresh_cache-34c75d7e-e63e-4eff-afad-0edd61e023fc" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1619.750482] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquired lock "refresh_cache-34c75d7e-e63e-4eff-afad-0edd61e023fc" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1619.750632] env[67752]: DEBUG nova.network.neutron [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1619.839066] env[67752]: DEBUG nova.network.neutron [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1620.285463] env[67752]: DEBUG nova.compute.manager [req-87d0669d-4604-43c8-881e-e0fa5cb29094 req-af297e95-ecf1-4d8e-8fdc-b2d5d4842214 service nova] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Received event network-vif-plugged-5682762b-1f55-416f-9dc9-47c9b0bd52c0 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1620.285684] env[67752]: DEBUG oslo_concurrency.lockutils [req-87d0669d-4604-43c8-881e-e0fa5cb29094 req-af297e95-ecf1-4d8e-8fdc-b2d5d4842214 service nova] Acquiring lock "34c75d7e-e63e-4eff-afad-0edd61e023fc-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1620.285917] env[67752]: DEBUG oslo_concurrency.lockutils [req-87d0669d-4604-43c8-881e-e0fa5cb29094 req-af297e95-ecf1-4d8e-8fdc-b2d5d4842214 service nova] Lock "34c75d7e-e63e-4eff-afad-0edd61e023fc-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1620.287264] env[67752]: DEBUG oslo_concurrency.lockutils [req-87d0669d-4604-43c8-881e-e0fa5cb29094 req-af297e95-ecf1-4d8e-8fdc-b2d5d4842214 service nova] Lock "34c75d7e-e63e-4eff-afad-0edd61e023fc-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1620.287422] env[67752]: DEBUG nova.compute.manager [req-87d0669d-4604-43c8-881e-e0fa5cb29094 req-af297e95-ecf1-4d8e-8fdc-b2d5d4842214 service nova] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] No waiting events found dispatching network-vif-plugged-5682762b-1f55-416f-9dc9-47c9b0bd52c0 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1620.287604] env[67752]: WARNING nova.compute.manager [req-87d0669d-4604-43c8-881e-e0fa5cb29094 req-af297e95-ecf1-4d8e-8fdc-b2d5d4842214 service nova] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Received unexpected event network-vif-plugged-5682762b-1f55-416f-9dc9-47c9b0bd52c0 for instance with vm_state building and task_state spawning. [ 1620.423296] env[67752]: DEBUG nova.network.neutron [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Updating instance_info_cache with network_info: [{"id": "5682762b-1f55-416f-9dc9-47c9b0bd52c0", "address": "fa:16:3e:a2:9c:51", "network": {"id": "48a676ef-ba7b-476c-8ee5-40eb99a49ec1", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-1953737432-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.13", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e117ebfe10cb4714b149d6e9bc132273", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "b36c5ae6-c344-4bd1-8239-29128e2bbfbf", "external-id": "nsx-vlan-transportzone-214", "segmentation_id": 214, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5682762b-1f", "ovs_interfaceid": "5682762b-1f55-416f-9dc9-47c9b0bd52c0", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1620.443744] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Releasing lock "refresh_cache-34c75d7e-e63e-4eff-afad-0edd61e023fc" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1620.443744] env[67752]: DEBUG nova.compute.manager [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Instance network_info: |[{"id": "5682762b-1f55-416f-9dc9-47c9b0bd52c0", "address": "fa:16:3e:a2:9c:51", "network": {"id": "48a676ef-ba7b-476c-8ee5-40eb99a49ec1", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-1953737432-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.13", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e117ebfe10cb4714b149d6e9bc132273", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "b36c5ae6-c344-4bd1-8239-29128e2bbfbf", "external-id": "nsx-vlan-transportzone-214", "segmentation_id": 214, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5682762b-1f", "ovs_interfaceid": "5682762b-1f55-416f-9dc9-47c9b0bd52c0", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1620.443744] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:a2:9c:51', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'b36c5ae6-c344-4bd1-8239-29128e2bbfbf', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '5682762b-1f55-416f-9dc9-47c9b0bd52c0', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1620.450186] env[67752]: DEBUG oslo.service.loopingcall [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1620.450763] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1620.450955] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-976d4d0f-0f21-43d4-a074-4798ad8df1bf {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1620.473419] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1620.473419] env[67752]: value = "task-3199801" [ 1620.473419] env[67752]: _type = "Task" [ 1620.473419] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1620.488021] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199801, 'name': CreateVM_Task} progress is 6%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1620.831743] env[67752]: DEBUG oslo_concurrency.lockutils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquiring lock "d52496eb-f8e3-4d86-8454-f56e9097777e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1620.832087] env[67752]: DEBUG oslo_concurrency.lockutils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "d52496eb-f8e3-4d86-8454-f56e9097777e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1620.984157] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199801, 'name': CreateVM_Task, 'duration_secs': 0.310246} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1620.984314] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1620.984932] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1620.985116] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1620.985451] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1620.985692] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-5c8ce8bd-d240-4a06-b821-ef14dae1e3ea {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1620.989985] env[67752]: DEBUG oslo_vmware.api [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Waiting for the task: (returnval){ [ 1620.989985] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52a4b564-f516-14c4-6650-368643214540" [ 1620.989985] env[67752]: _type = "Task" [ 1620.989985] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1620.997376] env[67752]: DEBUG oslo_vmware.api [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52a4b564-f516-14c4-6650-368643214540, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1621.500582] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1621.500866] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1621.501070] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1622.091866] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1622.315153] env[67752]: DEBUG nova.compute.manager [req-ae6901a8-1e17-4db8-bf51-95e556a2788f req-24c94813-8f49-4f90-b029-c8c10e131b5c service nova] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Received event network-changed-5682762b-1f55-416f-9dc9-47c9b0bd52c0 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1622.315422] env[67752]: DEBUG nova.compute.manager [req-ae6901a8-1e17-4db8-bf51-95e556a2788f req-24c94813-8f49-4f90-b029-c8c10e131b5c service nova] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Refreshing instance network info cache due to event network-changed-5682762b-1f55-416f-9dc9-47c9b0bd52c0. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1622.315653] env[67752]: DEBUG oslo_concurrency.lockutils [req-ae6901a8-1e17-4db8-bf51-95e556a2788f req-24c94813-8f49-4f90-b029-c8c10e131b5c service nova] Acquiring lock "refresh_cache-34c75d7e-e63e-4eff-afad-0edd61e023fc" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1622.315803] env[67752]: DEBUG oslo_concurrency.lockutils [req-ae6901a8-1e17-4db8-bf51-95e556a2788f req-24c94813-8f49-4f90-b029-c8c10e131b5c service nova] Acquired lock "refresh_cache-34c75d7e-e63e-4eff-afad-0edd61e023fc" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1622.316085] env[67752]: DEBUG nova.network.neutron [req-ae6901a8-1e17-4db8-bf51-95e556a2788f req-24c94813-8f49-4f90-b029-c8c10e131b5c service nova] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Refreshing network info cache for port 5682762b-1f55-416f-9dc9-47c9b0bd52c0 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1622.612396] env[67752]: DEBUG nova.network.neutron [req-ae6901a8-1e17-4db8-bf51-95e556a2788f req-24c94813-8f49-4f90-b029-c8c10e131b5c service nova] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Updated VIF entry in instance network info cache for port 5682762b-1f55-416f-9dc9-47c9b0bd52c0. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1622.612770] env[67752]: DEBUG nova.network.neutron [req-ae6901a8-1e17-4db8-bf51-95e556a2788f req-24c94813-8f49-4f90-b029-c8c10e131b5c service nova] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Updating instance_info_cache with network_info: [{"id": "5682762b-1f55-416f-9dc9-47c9b0bd52c0", "address": "fa:16:3e:a2:9c:51", "network": {"id": "48a676ef-ba7b-476c-8ee5-40eb99a49ec1", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-1953737432-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.13", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e117ebfe10cb4714b149d6e9bc132273", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "b36c5ae6-c344-4bd1-8239-29128e2bbfbf", "external-id": "nsx-vlan-transportzone-214", "segmentation_id": 214, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap5682762b-1f", "ovs_interfaceid": "5682762b-1f55-416f-9dc9-47c9b0bd52c0", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1622.623217] env[67752]: DEBUG oslo_concurrency.lockutils [req-ae6901a8-1e17-4db8-bf51-95e556a2788f req-24c94813-8f49-4f90-b029-c8c10e131b5c service nova] Releasing lock "refresh_cache-34c75d7e-e63e-4eff-afad-0edd61e023fc" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1623.635029] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1627.631701] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1627.656370] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1628.635341] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1628.635881] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 1628.636072] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 1628.655677] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1628.655820] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1628.655968] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1628.656132] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1628.656266] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1628.656388] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1628.656508] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1628.656629] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1628.656745] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1628.656861] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1628.656979] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 1628.657460] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1628.657641] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1629.635203] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1629.635391] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 1633.635701] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1633.647484] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1633.647722] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1633.647894] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1633.648096] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1633.649205] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19758452-6b45-4c18-9215-534996e47391 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1633.658070] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f42600e-758a-41fc-b7bc-4cc50ca276b2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1633.672971] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0ebeb97f-2586-4900-bf80-07adea843f7e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1633.680964] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4a31ec0b-faca-4773-bb21-f8c121a85caa {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1633.710075] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181006MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1633.710406] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1633.710646] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1633.783069] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1633.783236] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance a8bd310e-530a-46ab-add7-1b827ea5f399 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1633.783366] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9432cd04-c2df-428b-b31a-93d3ec0c9d1f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1633.783529] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 90e58033-eacd-433a-8d25-40f7d8403588 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1633.783657] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 81738fe6-6d50-4dfe-ae86-4c8771837168 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1633.783778] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e4ebb0f0-ab58-491f-bc4a-7548bbe415d8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1633.783894] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c1975c17-15a5-4751-bf7f-d5b7e0d2fea1 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1633.784017] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1633.784201] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 7557e3a1-b298-401f-8320-5a48ddda3298 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1633.784292] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 34c75d7e-e63e-4eff-afad-0edd61e023fc actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1633.796203] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1227bf67-5435-4c53-86ac-224e7e00d86c has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1633.806914] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 214ac8b8-c04c-47cf-8dfd-fba169d6ec44 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1633.816232] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 7ab65bf3-7bbd-4c70-8a6d-18691bba92b5 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1633.826194] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 048833ec-a7b2-4a8b-9204-788f2e9c5be7 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1633.835610] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d52496eb-f8e3-4d86-8454-f56e9097777e has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1633.835836] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1633.835983] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1634.009059] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aafdcf38-e3ff-4fe8-b1b9-dd63981f74b7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1634.016613] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-215cc0fb-fda9-44c7-8b4a-a89710e3b94d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1634.046413] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b51adce-f802-4437-b2e7-cfa286e79ee8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1634.053064] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c1a59bb8-e818-4c4a-ba61-e3d78b5f301f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1634.065539] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1634.074134] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1634.090581] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1634.090765] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.380s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1635.090735] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1636.674262] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7e62a619-0fef-4f36-87b8-86ef9d6b98fc tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "34c75d7e-e63e-4eff-afad-0edd61e023fc" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1649.039837] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "09e003b7-3c4c-4b11-a52d-749acf709068" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1649.039837] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "09e003b7-3c4c-4b11-a52d-749acf709068" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1658.110631] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Acquiring lock "fea66a16-92c3-4c7a-8bff-c6eb59270158" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1658.110967] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Lock "fea66a16-92c3-4c7a-8bff-c6eb59270158" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1663.396145] env[67752]: WARNING oslo_vmware.rw_handles [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1663.396145] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1663.396145] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1663.396145] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1663.396145] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1663.396145] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1663.396145] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1663.396145] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1663.396145] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1663.396145] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1663.396145] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1663.396145] env[67752]: ERROR oslo_vmware.rw_handles [ 1663.396806] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/04ccb8ff-deab-4fe9-b788-97dad60e8ef5/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1663.398893] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1663.399174] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Copying Virtual Disk [datastore2] vmware_temp/04ccb8ff-deab-4fe9-b788-97dad60e8ef5/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/04ccb8ff-deab-4fe9-b788-97dad60e8ef5/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1663.399463] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-089f8f81-8455-4fac-8194-7c1e5f56176e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1663.407617] env[67752]: DEBUG oslo_vmware.api [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Waiting for the task: (returnval){ [ 1663.407617] env[67752]: value = "task-3199802" [ 1663.407617] env[67752]: _type = "Task" [ 1663.407617] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1663.414887] env[67752]: DEBUG oslo_vmware.api [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Task: {'id': task-3199802, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1663.917863] env[67752]: DEBUG oslo_vmware.exceptions [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1663.918151] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1663.918699] env[67752]: ERROR nova.compute.manager [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1663.918699] env[67752]: Faults: ['InvalidArgument'] [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Traceback (most recent call last): [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] yield resources [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] self.driver.spawn(context, instance, image_meta, [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] self._fetch_image_if_missing(context, vi) [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] image_cache(vi, tmp_image_ds_loc) [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] vm_util.copy_virtual_disk( [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] session._wait_for_task(vmdk_copy_task) [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] return self.wait_for_task(task_ref) [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] return evt.wait() [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] result = hub.switch() [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] return self.greenlet.switch() [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] self.f(*self.args, **self.kw) [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] raise exceptions.translate_fault(task_info.error) [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Faults: ['InvalidArgument'] [ 1663.918699] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] [ 1663.919583] env[67752]: INFO nova.compute.manager [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Terminating instance [ 1663.920639] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1663.920847] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1663.921090] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-21b1c449-daaa-4931-9945-6b8373ef3a91 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1663.923161] env[67752]: DEBUG nova.compute.manager [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1663.923359] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1663.924068] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d2276af2-09ab-44d1-a7ec-bb01f4a38cd1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1663.930651] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1663.930855] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-5c5a6490-f486-4050-9f8f-18ee5bace7ec {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1663.932851] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1663.933034] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1663.933911] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-64619a21-cc68-42d2-958e-4de44feb6219 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1663.938288] env[67752]: DEBUG oslo_vmware.api [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Waiting for the task: (returnval){ [ 1663.938288] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52fa4147-962d-df8f-e63e-bffd18f9d8cf" [ 1663.938288] env[67752]: _type = "Task" [ 1663.938288] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1663.945546] env[67752]: DEBUG oslo_vmware.api [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52fa4147-962d-df8f-e63e-bffd18f9d8cf, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1664.005166] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1664.005426] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1664.005614] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Deleting the datastore file [datastore2] 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1664.005877] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-7d6199dd-499c-4c17-b88f-07a795a09b80 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1664.012064] env[67752]: DEBUG oslo_vmware.api [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Waiting for the task: (returnval){ [ 1664.012064] env[67752]: value = "task-3199804" [ 1664.012064] env[67752]: _type = "Task" [ 1664.012064] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1664.019339] env[67752]: DEBUG oslo_vmware.api [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Task: {'id': task-3199804, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1664.448709] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1664.449012] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Creating directory with path [datastore2] vmware_temp/dfd2424a-2b8e-4989-a92b-c648c7f9afa2/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1664.449267] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-c27288e4-72a2-4d55-ac52-610824817d72 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1664.460708] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Created directory with path [datastore2] vmware_temp/dfd2424a-2b8e-4989-a92b-c648c7f9afa2/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1664.460904] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Fetch image to [datastore2] vmware_temp/dfd2424a-2b8e-4989-a92b-c648c7f9afa2/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1664.461095] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/dfd2424a-2b8e-4989-a92b-c648c7f9afa2/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1664.461812] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c9275f49-1250-4cac-9629-1cad861c37f3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1664.468245] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f8e97ee-29f6-44fd-8e1c-637fa5d9f8c2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1664.476804] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f814d327-8adc-4e77-ba07-801b8bb919c6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1664.506979] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa55e0a1-eba6-403a-a11f-b5d0536fbac7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1664.512183] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-4b69d0f2-50d5-40f1-af6d-55a0cbac90f4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1664.520732] env[67752]: DEBUG oslo_vmware.api [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Task: {'id': task-3199804, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.065822} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1664.520956] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1664.521156] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1664.521330] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1664.521500] env[67752]: INFO nova.compute.manager [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1664.523576] env[67752]: DEBUG nova.compute.claims [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1664.523746] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1664.523960] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1664.538937] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1664.594191] env[67752]: DEBUG oslo_vmware.rw_handles [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/dfd2424a-2b8e-4989-a92b-c648c7f9afa2/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1664.654644] env[67752]: DEBUG oslo_vmware.rw_handles [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1664.654870] env[67752]: DEBUG oslo_vmware.rw_handles [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Closing write handle for https://esx7c2n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/dfd2424a-2b8e-4989-a92b-c648c7f9afa2/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1664.771483] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5de6a84c-47c7-4538-b955-cb64dbc65e49 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1664.779288] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4fe36d98-b7a6-4c2f-8cf2-c584c4bfd6b1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1664.808025] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0fbac66-63ae-4cc0-bfd9-27d10a6418be {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1664.814515] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-736a582e-41c5-4f9d-817a-f30460854a09 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1664.827013] env[67752]: DEBUG nova.compute.provider_tree [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1664.835399] env[67752]: DEBUG nova.scheduler.client.report [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1664.853257] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.329s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1664.853791] env[67752]: ERROR nova.compute.manager [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1664.853791] env[67752]: Faults: ['InvalidArgument'] [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Traceback (most recent call last): [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] self.driver.spawn(context, instance, image_meta, [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] self._fetch_image_if_missing(context, vi) [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] image_cache(vi, tmp_image_ds_loc) [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] vm_util.copy_virtual_disk( [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] session._wait_for_task(vmdk_copy_task) [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] return self.wait_for_task(task_ref) [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] return evt.wait() [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] result = hub.switch() [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] return self.greenlet.switch() [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] self.f(*self.args, **self.kw) [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] raise exceptions.translate_fault(task_info.error) [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Faults: ['InvalidArgument'] [ 1664.853791] env[67752]: ERROR nova.compute.manager [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] [ 1664.855739] env[67752]: DEBUG nova.compute.utils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1664.856511] env[67752]: DEBUG nova.compute.manager [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Build of instance 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05 was re-scheduled: A specified parameter was not correct: fileType [ 1664.856511] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1664.856909] env[67752]: DEBUG nova.compute.manager [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1664.858181] env[67752]: DEBUG nova.compute.manager [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1664.858181] env[67752]: DEBUG nova.compute.manager [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1664.858181] env[67752]: DEBUG nova.network.neutron [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1665.289708] env[67752]: DEBUG nova.network.neutron [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1665.304361] env[67752]: INFO nova.compute.manager [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Took 0.45 seconds to deallocate network for instance. [ 1665.389989] env[67752]: INFO nova.scheduler.client.report [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Deleted allocations for instance 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05 [ 1665.411169] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ebf7e428-3655-4425-8eee-d0c5b7cf9865 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "3d893e80-bbbb-4e58-bf1f-1edd4bccfd05" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 618.104s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1665.412324] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f615aab7-2485-44a2-9977-f6502eacc97f tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "3d893e80-bbbb-4e58-bf1f-1edd4bccfd05" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 421.499s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1665.412561] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f615aab7-2485-44a2-9977-f6502eacc97f tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "3d893e80-bbbb-4e58-bf1f-1edd4bccfd05-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1665.412762] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f615aab7-2485-44a2-9977-f6502eacc97f tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "3d893e80-bbbb-4e58-bf1f-1edd4bccfd05-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1665.412933] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f615aab7-2485-44a2-9977-f6502eacc97f tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "3d893e80-bbbb-4e58-bf1f-1edd4bccfd05-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1665.414974] env[67752]: INFO nova.compute.manager [None req-f615aab7-2485-44a2-9977-f6502eacc97f tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Terminating instance [ 1665.417394] env[67752]: DEBUG nova.compute.manager [None req-f615aab7-2485-44a2-9977-f6502eacc97f tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1665.417394] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f615aab7-2485-44a2-9977-f6502eacc97f tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1665.417861] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-f874282f-6a48-4c10-a65e-daae2af79fd0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1665.422837] env[67752]: DEBUG nova.compute.manager [None req-81605fee-f551-459c-a440-104dd375244b tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: 1227bf67-5435-4c53-86ac-224e7e00d86c] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1665.429077] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-666d05aa-dc66-4115-81bc-429d6d59dd6d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1665.457615] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-f615aab7-2485-44a2-9977-f6502eacc97f tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05 could not be found. [ 1665.457921] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f615aab7-2485-44a2-9977-f6502eacc97f tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1665.458133] env[67752]: INFO nova.compute.manager [None req-f615aab7-2485-44a2-9977-f6502eacc97f tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1665.458824] env[67752]: DEBUG oslo.service.loopingcall [None req-f615aab7-2485-44a2-9977-f6502eacc97f tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1665.458824] env[67752]: DEBUG nova.compute.manager [None req-81605fee-f551-459c-a440-104dd375244b tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: 1227bf67-5435-4c53-86ac-224e7e00d86c] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1665.459853] env[67752]: DEBUG nova.compute.manager [-] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1665.459853] env[67752]: DEBUG nova.network.neutron [-] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1665.483950] env[67752]: DEBUG oslo_concurrency.lockutils [None req-81605fee-f551-459c-a440-104dd375244b tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Lock "1227bf67-5435-4c53-86ac-224e7e00d86c" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 220.585s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1665.485402] env[67752]: DEBUG nova.network.neutron [-] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1665.494024] env[67752]: INFO nova.compute.manager [-] [instance: 3d893e80-bbbb-4e58-bf1f-1edd4bccfd05] Took 0.03 seconds to deallocate network for instance. [ 1665.494810] env[67752]: DEBUG nova.compute.manager [None req-2362a7e3-7d07-4116-8026-618dc9901d50 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 214ac8b8-c04c-47cf-8dfd-fba169d6ec44] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1665.519266] env[67752]: DEBUG nova.compute.manager [None req-2362a7e3-7d07-4116-8026-618dc9901d50 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 214ac8b8-c04c-47cf-8dfd-fba169d6ec44] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1665.539962] env[67752]: DEBUG oslo_concurrency.lockutils [None req-2362a7e3-7d07-4116-8026-618dc9901d50 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "214ac8b8-c04c-47cf-8dfd-fba169d6ec44" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 212.249s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1665.550406] env[67752]: DEBUG nova.compute.manager [None req-ac5a34f5-4287-4338-8801-fc5e30e196ac tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] [instance: 7ab65bf3-7bbd-4c70-8a6d-18691bba92b5] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1665.580635] env[67752]: DEBUG nova.compute.manager [None req-ac5a34f5-4287-4338-8801-fc5e30e196ac tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] [instance: 7ab65bf3-7bbd-4c70-8a6d-18691bba92b5] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1665.596947] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f615aab7-2485-44a2-9977-f6502eacc97f tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "3d893e80-bbbb-4e58-bf1f-1edd4bccfd05" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.185s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1665.601910] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ac5a34f5-4287-4338-8801-fc5e30e196ac tempest-ServerDiskConfigTestJSON-1491062674 tempest-ServerDiskConfigTestJSON-1491062674-project-member] Lock "7ab65bf3-7bbd-4c70-8a6d-18691bba92b5" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 211.238s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1665.610623] env[67752]: DEBUG nova.compute.manager [None req-4d9008c8-daab-45ba-856a-ef4fde4dc3c5 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: 048833ec-a7b2-4a8b-9204-788f2e9c5be7] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1665.641525] env[67752]: DEBUG nova.compute.manager [None req-4d9008c8-daab-45ba-856a-ef4fde4dc3c5 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: 048833ec-a7b2-4a8b-9204-788f2e9c5be7] Instance disappeared before build. {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2430}} [ 1665.662433] env[67752]: DEBUG oslo_concurrency.lockutils [None req-4d9008c8-daab-45ba-856a-ef4fde4dc3c5 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Lock "048833ec-a7b2-4a8b-9204-788f2e9c5be7" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 205.569s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1665.670840] env[67752]: DEBUG nova.compute.manager [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1665.718563] env[67752]: DEBUG oslo_concurrency.lockutils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1665.718844] env[67752]: DEBUG oslo_concurrency.lockutils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1665.720412] env[67752]: INFO nova.compute.claims [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1665.900489] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-318fb220-9476-45e5-b588-34cd087578fb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1665.908049] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ede275bc-e989-4e3e-a9ca-71780047cd9c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1665.936439] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56321bef-0adf-4538-bfe7-b597aad0ddad {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1665.942896] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b5e3d2e9-e70e-4f3a-9aa0-f7af36b89fe0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1665.955666] env[67752]: DEBUG nova.compute.provider_tree [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1665.964320] env[67752]: DEBUG nova.scheduler.client.report [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1665.977596] env[67752]: DEBUG oslo_concurrency.lockutils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.259s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1665.978088] env[67752]: DEBUG nova.compute.manager [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1666.008192] env[67752]: DEBUG nova.compute.utils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1666.009522] env[67752]: DEBUG nova.compute.manager [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1666.009693] env[67752]: DEBUG nova.network.neutron [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1666.017825] env[67752]: DEBUG nova.compute.manager [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1666.077788] env[67752]: DEBUG nova.compute.manager [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1666.081284] env[67752]: DEBUG nova.policy [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'f31e3565acaa4672acaac19c5214b28e', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e18ee59529f846859b4f8b8ed92852ed', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1666.102932] env[67752]: DEBUG nova.virt.hardware [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1666.103194] env[67752]: DEBUG nova.virt.hardware [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1666.103354] env[67752]: DEBUG nova.virt.hardware [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1666.103538] env[67752]: DEBUG nova.virt.hardware [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1666.103686] env[67752]: DEBUG nova.virt.hardware [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1666.103835] env[67752]: DEBUG nova.virt.hardware [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1666.104055] env[67752]: DEBUG nova.virt.hardware [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1666.104227] env[67752]: DEBUG nova.virt.hardware [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1666.104398] env[67752]: DEBUG nova.virt.hardware [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1666.104562] env[67752]: DEBUG nova.virt.hardware [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1666.104760] env[67752]: DEBUG nova.virt.hardware [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1666.105669] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f8a9000c-023a-45f8-bc07-1fb929319f0f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1666.113448] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5dcfd1e4-9b5d-42e9-b186-c8c601f1f5d3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1666.765305] env[67752]: DEBUG nova.network.neutron [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Successfully created port: 36caaa65-7122-48b4-ad3a-42bf985dd7e5 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1667.111995] env[67752]: DEBUG nova.network.neutron [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Successfully created port: b7454e12-5aad-42db-b26c-5d8d324d209d {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1667.757783] env[67752]: DEBUG nova.compute.manager [req-443e6d2a-81cc-4699-9858-09e754898339 req-25b8f364-3057-4434-9835-b9cc79b7efaf service nova] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Received event network-vif-plugged-36caaa65-7122-48b4-ad3a-42bf985dd7e5 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1667.758228] env[67752]: DEBUG oslo_concurrency.lockutils [req-443e6d2a-81cc-4699-9858-09e754898339 req-25b8f364-3057-4434-9835-b9cc79b7efaf service nova] Acquiring lock "d52496eb-f8e3-4d86-8454-f56e9097777e-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1667.758491] env[67752]: DEBUG oslo_concurrency.lockutils [req-443e6d2a-81cc-4699-9858-09e754898339 req-25b8f364-3057-4434-9835-b9cc79b7efaf service nova] Lock "d52496eb-f8e3-4d86-8454-f56e9097777e-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1667.758673] env[67752]: DEBUG oslo_concurrency.lockutils [req-443e6d2a-81cc-4699-9858-09e754898339 req-25b8f364-3057-4434-9835-b9cc79b7efaf service nova] Lock "d52496eb-f8e3-4d86-8454-f56e9097777e-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1667.758892] env[67752]: DEBUG nova.compute.manager [req-443e6d2a-81cc-4699-9858-09e754898339 req-25b8f364-3057-4434-9835-b9cc79b7efaf service nova] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] No waiting events found dispatching network-vif-plugged-36caaa65-7122-48b4-ad3a-42bf985dd7e5 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1667.759024] env[67752]: WARNING nova.compute.manager [req-443e6d2a-81cc-4699-9858-09e754898339 req-25b8f364-3057-4434-9835-b9cc79b7efaf service nova] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Received unexpected event network-vif-plugged-36caaa65-7122-48b4-ad3a-42bf985dd7e5 for instance with vm_state building and task_state spawning. [ 1667.877601] env[67752]: DEBUG nova.network.neutron [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Successfully updated port: 36caaa65-7122-48b4-ad3a-42bf985dd7e5 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1668.647623] env[67752]: DEBUG nova.network.neutron [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Successfully updated port: b7454e12-5aad-42db-b26c-5d8d324d209d {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1668.656196] env[67752]: DEBUG oslo_concurrency.lockutils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquiring lock "refresh_cache-d52496eb-f8e3-4d86-8454-f56e9097777e" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1668.656505] env[67752]: DEBUG oslo_concurrency.lockutils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquired lock "refresh_cache-d52496eb-f8e3-4d86-8454-f56e9097777e" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1668.656505] env[67752]: DEBUG nova.network.neutron [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1668.714046] env[67752]: DEBUG nova.network.neutron [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1669.118273] env[67752]: DEBUG nova.network.neutron [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Updating instance_info_cache with network_info: [{"id": "36caaa65-7122-48b4-ad3a-42bf985dd7e5", "address": "fa:16:3e:c6:b6:30", "network": {"id": "519ec02d-93b3-4eeb-b2d2-c63ee870d9b4", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-2089981631", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.249", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e18ee59529f846859b4f8b8ed92852ed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "7d377d75-3add-4a15-8691-74b2eb010924", "external-id": "nsx-vlan-transportzone-71", "segmentation_id": 71, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap36caaa65-71", "ovs_interfaceid": "36caaa65-7122-48b4-ad3a-42bf985dd7e5", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "b7454e12-5aad-42db-b26c-5d8d324d209d", "address": "fa:16:3e:00:e2:1d", "network": {"id": "593a667a-e762-4a7e-becf-822857256f42", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-12373730", "subnets": [{"cidr": "192.168.129.0/24", "dns": [], "gateway": {"address": "192.168.129.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.129.15", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.129.2"}}], "meta": {"injected": false, "tenant_id": "e18ee59529f846859b4f8b8ed92852ed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "fb224918-e294-4b76-80f9-2fa0031b7dc2", "external-id": "nsx-vlan-transportzone-876", "segmentation_id": 876, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapb7454e12-5a", "ovs_interfaceid": "b7454e12-5aad-42db-b26c-5d8d324d209d", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1669.131357] env[67752]: DEBUG oslo_concurrency.lockutils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Releasing lock "refresh_cache-d52496eb-f8e3-4d86-8454-f56e9097777e" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1669.131685] env[67752]: DEBUG nova.compute.manager [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Instance network_info: |[{"id": "36caaa65-7122-48b4-ad3a-42bf985dd7e5", "address": "fa:16:3e:c6:b6:30", "network": {"id": "519ec02d-93b3-4eeb-b2d2-c63ee870d9b4", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-2089981631", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.249", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e18ee59529f846859b4f8b8ed92852ed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "7d377d75-3add-4a15-8691-74b2eb010924", "external-id": "nsx-vlan-transportzone-71", "segmentation_id": 71, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap36caaa65-71", "ovs_interfaceid": "36caaa65-7122-48b4-ad3a-42bf985dd7e5", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "b7454e12-5aad-42db-b26c-5d8d324d209d", "address": "fa:16:3e:00:e2:1d", "network": {"id": "593a667a-e762-4a7e-becf-822857256f42", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-12373730", "subnets": [{"cidr": "192.168.129.0/24", "dns": [], "gateway": {"address": "192.168.129.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.129.15", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.129.2"}}], "meta": {"injected": false, "tenant_id": "e18ee59529f846859b4f8b8ed92852ed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "fb224918-e294-4b76-80f9-2fa0031b7dc2", "external-id": "nsx-vlan-transportzone-876", "segmentation_id": 876, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapb7454e12-5a", "ovs_interfaceid": "b7454e12-5aad-42db-b26c-5d8d324d209d", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1669.132137] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:c6:b6:30', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '7d377d75-3add-4a15-8691-74b2eb010924', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '36caaa65-7122-48b4-ad3a-42bf985dd7e5', 'vif_model': 'vmxnet3'}, {'network_name': 'br-int', 'mac_address': 'fa:16:3e:00:e2:1d', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'fb224918-e294-4b76-80f9-2fa0031b7dc2', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'b7454e12-5aad-42db-b26c-5d8d324d209d', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1669.141334] env[67752]: DEBUG oslo.service.loopingcall [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1669.141823] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1669.142069] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-68ce474c-afa1-40f3-9702-f1ce092bb237 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1669.163722] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1669.163722] env[67752]: value = "task-3199805" [ 1669.163722] env[67752]: _type = "Task" [ 1669.163722] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1669.171312] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199805, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1669.674715] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199805, 'name': CreateVM_Task, 'duration_secs': 0.334193} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1669.674920] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1669.675710] env[67752]: DEBUG oslo_concurrency.lockutils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1669.675877] env[67752]: DEBUG oslo_concurrency.lockutils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1669.676254] env[67752]: DEBUG oslo_concurrency.lockutils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1669.676462] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-2392b7dd-aefa-4911-91d3-4a449df17168 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1669.680940] env[67752]: DEBUG oslo_vmware.api [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Waiting for the task: (returnval){ [ 1669.680940] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]522ae38b-3c4d-9593-2729-89c4f05d30b0" [ 1669.680940] env[67752]: _type = "Task" [ 1669.680940] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1669.688442] env[67752]: DEBUG oslo_vmware.api [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]522ae38b-3c4d-9593-2729-89c4f05d30b0, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1669.784432] env[67752]: DEBUG nova.compute.manager [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Received event network-changed-36caaa65-7122-48b4-ad3a-42bf985dd7e5 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1669.784432] env[67752]: DEBUG nova.compute.manager [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Refreshing instance network info cache due to event network-changed-36caaa65-7122-48b4-ad3a-42bf985dd7e5. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1669.784432] env[67752]: DEBUG oslo_concurrency.lockutils [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] Acquiring lock "refresh_cache-d52496eb-f8e3-4d86-8454-f56e9097777e" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1669.784432] env[67752]: DEBUG oslo_concurrency.lockutils [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] Acquired lock "refresh_cache-d52496eb-f8e3-4d86-8454-f56e9097777e" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1669.784432] env[67752]: DEBUG nova.network.neutron [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Refreshing network info cache for port 36caaa65-7122-48b4-ad3a-42bf985dd7e5 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1670.038538] env[67752]: DEBUG nova.network.neutron [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Updated VIF entry in instance network info cache for port 36caaa65-7122-48b4-ad3a-42bf985dd7e5. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1670.039015] env[67752]: DEBUG nova.network.neutron [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Updating instance_info_cache with network_info: [{"id": "36caaa65-7122-48b4-ad3a-42bf985dd7e5", "address": "fa:16:3e:c6:b6:30", "network": {"id": "519ec02d-93b3-4eeb-b2d2-c63ee870d9b4", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-2089981631", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.249", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e18ee59529f846859b4f8b8ed92852ed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "7d377d75-3add-4a15-8691-74b2eb010924", "external-id": "nsx-vlan-transportzone-71", "segmentation_id": 71, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap36caaa65-71", "ovs_interfaceid": "36caaa65-7122-48b4-ad3a-42bf985dd7e5", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "b7454e12-5aad-42db-b26c-5d8d324d209d", "address": "fa:16:3e:00:e2:1d", "network": {"id": "593a667a-e762-4a7e-becf-822857256f42", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-12373730", "subnets": [{"cidr": "192.168.129.0/24", "dns": [], "gateway": {"address": "192.168.129.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.129.15", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.129.2"}}], "meta": {"injected": false, "tenant_id": "e18ee59529f846859b4f8b8ed92852ed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "fb224918-e294-4b76-80f9-2fa0031b7dc2", "external-id": "nsx-vlan-transportzone-876", "segmentation_id": 876, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapb7454e12-5a", "ovs_interfaceid": "b7454e12-5aad-42db-b26c-5d8d324d209d", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1670.048844] env[67752]: DEBUG oslo_concurrency.lockutils [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] Releasing lock "refresh_cache-d52496eb-f8e3-4d86-8454-f56e9097777e" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1670.048988] env[67752]: DEBUG nova.compute.manager [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Received event network-vif-plugged-b7454e12-5aad-42db-b26c-5d8d324d209d {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1670.049197] env[67752]: DEBUG oslo_concurrency.lockutils [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] Acquiring lock "d52496eb-f8e3-4d86-8454-f56e9097777e-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1670.049402] env[67752]: DEBUG oslo_concurrency.lockutils [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] Lock "d52496eb-f8e3-4d86-8454-f56e9097777e-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1670.049568] env[67752]: DEBUG oslo_concurrency.lockutils [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] Lock "d52496eb-f8e3-4d86-8454-f56e9097777e-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1670.049735] env[67752]: DEBUG nova.compute.manager [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] No waiting events found dispatching network-vif-plugged-b7454e12-5aad-42db-b26c-5d8d324d209d {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1670.049904] env[67752]: WARNING nova.compute.manager [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Received unexpected event network-vif-plugged-b7454e12-5aad-42db-b26c-5d8d324d209d for instance with vm_state building and task_state spawning. [ 1670.050084] env[67752]: DEBUG nova.compute.manager [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Received event network-changed-b7454e12-5aad-42db-b26c-5d8d324d209d {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1670.050245] env[67752]: DEBUG nova.compute.manager [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Refreshing instance network info cache due to event network-changed-b7454e12-5aad-42db-b26c-5d8d324d209d. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1670.050424] env[67752]: DEBUG oslo_concurrency.lockutils [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] Acquiring lock "refresh_cache-d52496eb-f8e3-4d86-8454-f56e9097777e" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1670.050563] env[67752]: DEBUG oslo_concurrency.lockutils [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] Acquired lock "refresh_cache-d52496eb-f8e3-4d86-8454-f56e9097777e" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1670.050718] env[67752]: DEBUG nova.network.neutron [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Refreshing network info cache for port b7454e12-5aad-42db-b26c-5d8d324d209d {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1670.192548] env[67752]: DEBUG oslo_concurrency.lockutils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1670.192949] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1670.193020] env[67752]: DEBUG oslo_concurrency.lockutils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1670.633515] env[67752]: DEBUG nova.network.neutron [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Updated VIF entry in instance network info cache for port b7454e12-5aad-42db-b26c-5d8d324d209d. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1670.633938] env[67752]: DEBUG nova.network.neutron [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Updating instance_info_cache with network_info: [{"id": "36caaa65-7122-48b4-ad3a-42bf985dd7e5", "address": "fa:16:3e:c6:b6:30", "network": {"id": "519ec02d-93b3-4eeb-b2d2-c63ee870d9b4", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-2089981631", "subnets": [{"cidr": "192.168.128.0/24", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.249", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e18ee59529f846859b4f8b8ed92852ed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "7d377d75-3add-4a15-8691-74b2eb010924", "external-id": "nsx-vlan-transportzone-71", "segmentation_id": 71, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap36caaa65-71", "ovs_interfaceid": "36caaa65-7122-48b4-ad3a-42bf985dd7e5", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}, {"id": "b7454e12-5aad-42db-b26c-5d8d324d209d", "address": "fa:16:3e:00:e2:1d", "network": {"id": "593a667a-e762-4a7e-becf-822857256f42", "bridge": "br-int", "label": "tempest-ServersTestMultiNic-12373730", "subnets": [{"cidr": "192.168.129.0/24", "dns": [], "gateway": {"address": "192.168.129.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.129.15", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.129.2"}}], "meta": {"injected": false, "tenant_id": "e18ee59529f846859b4f8b8ed92852ed", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "fb224918-e294-4b76-80f9-2fa0031b7dc2", "external-id": "nsx-vlan-transportzone-876", "segmentation_id": 876, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapb7454e12-5a", "ovs_interfaceid": "b7454e12-5aad-42db-b26c-5d8d324d209d", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1670.644285] env[67752]: DEBUG oslo_concurrency.lockutils [req-b87e7107-30b7-489e-ad6c-26ac8eafcda9 req-6294fbad-3a11-4006-9173-a12bd970782a service nova] Releasing lock "refresh_cache-d52496eb-f8e3-4d86-8454-f56e9097777e" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1682.630267] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1685.634635] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1688.635631] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1689.634528] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1689.634796] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1689.635018] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1689.635222] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 1690.636996] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1690.637331] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 1690.637331] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 1690.660162] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1690.660380] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1690.660443] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1690.660569] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1690.660687] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1690.660810] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1690.660931] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1690.661071] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1690.661195] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1690.661317] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1690.661437] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 1694.634699] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1694.635201] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1694.647027] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1694.647256] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1694.647428] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1694.647610] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1694.648718] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3fa2c54c-34ff-44c7-8336-a1b4028d179e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1694.657542] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-147ca2e8-5e99-41bc-843e-eb5b0f5ec1c6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1694.671083] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-06ccdca1-f158-42b2-b57e-3dd303af5c59 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1694.677120] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5979fec2-2774-4207-94e0-810856f12502 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1694.705631] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181010MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1694.705804] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1694.705998] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1694.777649] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance a8bd310e-530a-46ab-add7-1b827ea5f399 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1694.777811] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9432cd04-c2df-428b-b31a-93d3ec0c9d1f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1694.777940] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 90e58033-eacd-433a-8d25-40f7d8403588 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1694.778081] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 81738fe6-6d50-4dfe-ae86-4c8771837168 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1694.778208] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e4ebb0f0-ab58-491f-bc4a-7548bbe415d8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1694.778330] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c1975c17-15a5-4751-bf7f-d5b7e0d2fea1 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1694.778450] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1694.778594] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 7557e3a1-b298-401f-8320-5a48ddda3298 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1694.778716] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 34c75d7e-e63e-4eff-afad-0edd61e023fc actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1694.778834] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d52496eb-f8e3-4d86-8454-f56e9097777e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1694.789995] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 09e003b7-3c4c-4b11-a52d-749acf709068 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1694.800900] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fea66a16-92c3-4c7a-8bff-c6eb59270158 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1694.801227] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1694.801483] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1694.943260] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75923a82-fa1c-4103-8b64-253e66207ab6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1694.950851] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-58f27ea2-a10c-4dbc-9c70-14e4c5dc9079 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1694.980398] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-36acfc1c-c63e-48dd-9029-86062e462828 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1694.987329] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50dd534d-773e-4a31-b17d-6a089ebeec3b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1694.999803] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1695.008070] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1695.022740] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1695.022922] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.317s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1713.780224] env[67752]: WARNING oslo_vmware.rw_handles [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1713.780224] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1713.780224] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1713.780224] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1713.780224] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1713.780224] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1713.780224] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1713.780224] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1713.780224] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1713.780224] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1713.780224] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1713.780224] env[67752]: ERROR oslo_vmware.rw_handles [ 1713.780835] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/dfd2424a-2b8e-4989-a92b-c648c7f9afa2/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1713.782804] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1713.783061] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Copying Virtual Disk [datastore2] vmware_temp/dfd2424a-2b8e-4989-a92b-c648c7f9afa2/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/dfd2424a-2b8e-4989-a92b-c648c7f9afa2/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1713.783348] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-cc0b9f7a-a578-47a5-99eb-08b08089e836 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1713.790987] env[67752]: DEBUG oslo_vmware.api [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Waiting for the task: (returnval){ [ 1713.790987] env[67752]: value = "task-3199806" [ 1713.790987] env[67752]: _type = "Task" [ 1713.790987] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1713.799678] env[67752]: DEBUG oslo_vmware.api [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Task: {'id': task-3199806, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1714.301467] env[67752]: DEBUG oslo_vmware.exceptions [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1714.301744] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1714.302346] env[67752]: ERROR nova.compute.manager [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1714.302346] env[67752]: Faults: ['InvalidArgument'] [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Traceback (most recent call last): [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] yield resources [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] self.driver.spawn(context, instance, image_meta, [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] self._fetch_image_if_missing(context, vi) [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] image_cache(vi, tmp_image_ds_loc) [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] vm_util.copy_virtual_disk( [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] session._wait_for_task(vmdk_copy_task) [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] return self.wait_for_task(task_ref) [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] return evt.wait() [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] result = hub.switch() [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] return self.greenlet.switch() [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] self.f(*self.args, **self.kw) [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] raise exceptions.translate_fault(task_info.error) [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Faults: ['InvalidArgument'] [ 1714.302346] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] [ 1714.303284] env[67752]: INFO nova.compute.manager [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Terminating instance [ 1714.304205] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1714.304413] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1714.304640] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-6c36307e-3bd7-4e78-90be-21646362f2b2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1714.306856] env[67752]: DEBUG nova.compute.manager [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1714.307064] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1714.307876] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ab2f4650-e343-4e70-9811-4c5acd22d435 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1714.315574] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1714.315834] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-cb50fbbb-5fed-4ffc-865d-4ef422022493 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1714.318458] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1714.318701] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1714.319769] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-1840e922-8186-4d53-b03f-cf5e1f4c1838 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1714.324832] env[67752]: DEBUG oslo_vmware.api [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Waiting for the task: (returnval){ [ 1714.324832] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52fa22da-ea0e-8bc5-e6a0-46c7c13d9ffb" [ 1714.324832] env[67752]: _type = "Task" [ 1714.324832] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1714.333268] env[67752]: DEBUG oslo_vmware.api [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52fa22da-ea0e-8bc5-e6a0-46c7c13d9ffb, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1714.391028] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1714.391981] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1714.391981] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Deleting the datastore file [datastore2] a8bd310e-530a-46ab-add7-1b827ea5f399 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1714.391981] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-33328237-ab2e-4b7e-a50f-9884394c22d9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1714.397442] env[67752]: DEBUG oslo_vmware.api [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Waiting for the task: (returnval){ [ 1714.397442] env[67752]: value = "task-3199808" [ 1714.397442] env[67752]: _type = "Task" [ 1714.397442] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1714.405101] env[67752]: DEBUG oslo_vmware.api [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Task: {'id': task-3199808, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1714.835318] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1714.835669] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Creating directory with path [datastore2] vmware_temp/37db4d0f-58c0-402c-aa44-9ee10fa51a33/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1714.835877] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-43916a0f-d7bb-4b92-9db8-aeb31493fb83 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1714.846978] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Created directory with path [datastore2] vmware_temp/37db4d0f-58c0-402c-aa44-9ee10fa51a33/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1714.847202] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Fetch image to [datastore2] vmware_temp/37db4d0f-58c0-402c-aa44-9ee10fa51a33/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1714.847374] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/37db4d0f-58c0-402c-aa44-9ee10fa51a33/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1714.848124] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b8e30152-1c8d-4319-a93f-9c361ff4add1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1714.854510] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-63786699-ab51-4a7e-9627-997c7b21072f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1714.863387] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b0309db0-c859-47a5-95e8-174325d9b5b9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1714.893581] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-028acac9-0d38-4ccb-b396-16239f2ef193 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1714.901604] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-414a0357-94e4-493e-bce2-ad95df8635e2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1714.908642] env[67752]: DEBUG oslo_vmware.api [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Task: {'id': task-3199808, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.069447} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1714.908860] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1714.909098] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1714.909284] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1714.909457] env[67752]: INFO nova.compute.manager [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1714.911513] env[67752]: DEBUG nova.compute.claims [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1714.911685] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1714.911897] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1714.923998] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1715.067139] env[67752]: DEBUG oslo_vmware.rw_handles [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/37db4d0f-58c0-402c-aa44-9ee10fa51a33/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1715.129083] env[67752]: DEBUG oslo_vmware.rw_handles [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1715.129287] env[67752]: DEBUG oslo_vmware.rw_handles [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Closing write handle for https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/37db4d0f-58c0-402c-aa44-9ee10fa51a33/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1715.159176] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5a0f4860-67fc-4ca4-a3bd-e0dea796dd46 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1715.166365] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4124a2a7-b782-4429-a70f-c9bb82abdb69 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1715.195528] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db75022a-6442-402f-ac77-cd087ad2cfb0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1715.202295] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f3264e07-a4f0-41cd-a12c-7537724bf26d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1715.214686] env[67752]: DEBUG nova.compute.provider_tree [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1715.223079] env[67752]: DEBUG nova.scheduler.client.report [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1715.236346] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.324s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1715.236879] env[67752]: ERROR nova.compute.manager [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1715.236879] env[67752]: Faults: ['InvalidArgument'] [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Traceback (most recent call last): [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] self.driver.spawn(context, instance, image_meta, [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] self._fetch_image_if_missing(context, vi) [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] image_cache(vi, tmp_image_ds_loc) [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] vm_util.copy_virtual_disk( [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] session._wait_for_task(vmdk_copy_task) [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] return self.wait_for_task(task_ref) [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] return evt.wait() [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] result = hub.switch() [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] return self.greenlet.switch() [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] self.f(*self.args, **self.kw) [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] raise exceptions.translate_fault(task_info.error) [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Faults: ['InvalidArgument'] [ 1715.236879] env[67752]: ERROR nova.compute.manager [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] [ 1715.237920] env[67752]: DEBUG nova.compute.utils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1715.238958] env[67752]: DEBUG nova.compute.manager [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Build of instance a8bd310e-530a-46ab-add7-1b827ea5f399 was re-scheduled: A specified parameter was not correct: fileType [ 1715.238958] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1715.239337] env[67752]: DEBUG nova.compute.manager [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1715.239512] env[67752]: DEBUG nova.compute.manager [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1715.239684] env[67752]: DEBUG nova.compute.manager [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1715.239849] env[67752]: DEBUG nova.network.neutron [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1715.555305] env[67752]: DEBUG nova.network.neutron [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1715.570155] env[67752]: INFO nova.compute.manager [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Took 0.33 seconds to deallocate network for instance. [ 1715.681617] env[67752]: INFO nova.scheduler.client.report [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Deleted allocations for instance a8bd310e-530a-46ab-add7-1b827ea5f399 [ 1715.707620] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d740eae2-7ed9-4e65-8c01-de900f36e5ad tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Lock "a8bd310e-530a-46ab-add7-1b827ea5f399" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 664.135s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1715.709911] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0c67db20-a74a-4011-8412-800f0084d087 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Lock "a8bd310e-530a-46ab-add7-1b827ea5f399" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 468.392s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1715.709911] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0c67db20-a74a-4011-8412-800f0084d087 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Acquiring lock "a8bd310e-530a-46ab-add7-1b827ea5f399-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1715.709911] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0c67db20-a74a-4011-8412-800f0084d087 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Lock "a8bd310e-530a-46ab-add7-1b827ea5f399-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1715.710113] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0c67db20-a74a-4011-8412-800f0084d087 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Lock "a8bd310e-530a-46ab-add7-1b827ea5f399-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1715.712145] env[67752]: INFO nova.compute.manager [None req-0c67db20-a74a-4011-8412-800f0084d087 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Terminating instance [ 1715.714473] env[67752]: DEBUG nova.compute.manager [None req-0c67db20-a74a-4011-8412-800f0084d087 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1715.714987] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0c67db20-a74a-4011-8412-800f0084d087 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1715.715184] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-507c7232-c95b-453c-a169-7d553834319b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1715.720359] env[67752]: DEBUG nova.compute.manager [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1715.727507] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-500a1580-8b20-4128-9b7f-c36bb601166b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1715.757048] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-0c67db20-a74a-4011-8412-800f0084d087 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance a8bd310e-530a-46ab-add7-1b827ea5f399 could not be found. [ 1715.757274] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-0c67db20-a74a-4011-8412-800f0084d087 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1715.757565] env[67752]: INFO nova.compute.manager [None req-0c67db20-a74a-4011-8412-800f0084d087 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1715.757813] env[67752]: DEBUG oslo.service.loopingcall [None req-0c67db20-a74a-4011-8412-800f0084d087 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1715.762504] env[67752]: DEBUG nova.compute.manager [-] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1715.762620] env[67752]: DEBUG nova.network.neutron [-] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1715.775723] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1715.775987] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1715.777430] env[67752]: INFO nova.compute.claims [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1715.788109] env[67752]: DEBUG nova.network.neutron [-] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1715.797771] env[67752]: INFO nova.compute.manager [-] [instance: a8bd310e-530a-46ab-add7-1b827ea5f399] Took 0.04 seconds to deallocate network for instance. [ 1715.891425] env[67752]: DEBUG oslo_concurrency.lockutils [None req-0c67db20-a74a-4011-8412-800f0084d087 tempest-ImagesTestJSON-1113590265 tempest-ImagesTestJSON-1113590265-project-member] Lock "a8bd310e-530a-46ab-add7-1b827ea5f399" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.182s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1715.969492] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-72a088d8-4893-4b49-a4e3-fece5cec795b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1715.977535] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-231c9db4-ca4b-453f-a858-33032509bbcd {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1716.007909] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-18cd530d-811e-4706-8f1e-c129e4f0b906 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1716.016014] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8da4014c-05ba-4c91-9ea3-1995da9f7044 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1716.028190] env[67752]: DEBUG nova.compute.provider_tree [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1716.037783] env[67752]: DEBUG nova.scheduler.client.report [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1716.052134] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.276s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1716.052610] env[67752]: DEBUG nova.compute.manager [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1716.089014] env[67752]: DEBUG nova.compute.utils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1716.090437] env[67752]: DEBUG nova.compute.manager [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1716.090639] env[67752]: DEBUG nova.network.neutron [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1716.100317] env[67752]: DEBUG nova.compute.manager [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1716.150966] env[67752]: DEBUG nova.policy [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7ceddc353fd444808a8e3fa9a98925c8', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '17776dc4e25145179ebfbdfd4a9639f8', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1716.165115] env[67752]: DEBUG nova.compute.manager [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1716.190944] env[67752]: DEBUG nova.virt.hardware [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1716.191223] env[67752]: DEBUG nova.virt.hardware [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1716.191399] env[67752]: DEBUG nova.virt.hardware [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1716.191572] env[67752]: DEBUG nova.virt.hardware [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1716.191720] env[67752]: DEBUG nova.virt.hardware [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1716.191867] env[67752]: DEBUG nova.virt.hardware [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1716.192082] env[67752]: DEBUG nova.virt.hardware [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1716.192244] env[67752]: DEBUG nova.virt.hardware [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1716.192412] env[67752]: DEBUG nova.virt.hardware [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1716.192572] env[67752]: DEBUG nova.virt.hardware [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1716.192746] env[67752]: DEBUG nova.virt.hardware [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1716.193607] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e6f4a88-75de-47e9-aa29-a902e85b8433 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1716.201522] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1aa89d63-5f6c-48bd-a248-bfe191a0b8c4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1716.663682] env[67752]: DEBUG nova.network.neutron [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Successfully created port: 0be69bd5-bbce-42e8-a36f-d8bf7ee56d35 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1717.481152] env[67752]: DEBUG nova.network.neutron [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Successfully updated port: 0be69bd5-bbce-42e8-a36f-d8bf7ee56d35 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1717.491736] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "refresh_cache-09e003b7-3c4c-4b11-a52d-749acf709068" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1717.491892] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquired lock "refresh_cache-09e003b7-3c4c-4b11-a52d-749acf709068" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1717.492194] env[67752]: DEBUG nova.network.neutron [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1717.533098] env[67752]: DEBUG nova.network.neutron [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1717.597131] env[67752]: DEBUG nova.compute.manager [req-bae7e9a4-513e-41e6-987e-933ea75ca6db req-8e277f4c-30cb-4bb6-9723-f0edbdff3a5a service nova] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Received event network-vif-plugged-0be69bd5-bbce-42e8-a36f-d8bf7ee56d35 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1717.597429] env[67752]: DEBUG oslo_concurrency.lockutils [req-bae7e9a4-513e-41e6-987e-933ea75ca6db req-8e277f4c-30cb-4bb6-9723-f0edbdff3a5a service nova] Acquiring lock "09e003b7-3c4c-4b11-a52d-749acf709068-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1717.597697] env[67752]: DEBUG oslo_concurrency.lockutils [req-bae7e9a4-513e-41e6-987e-933ea75ca6db req-8e277f4c-30cb-4bb6-9723-f0edbdff3a5a service nova] Lock "09e003b7-3c4c-4b11-a52d-749acf709068-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1717.597883] env[67752]: DEBUG oslo_concurrency.lockutils [req-bae7e9a4-513e-41e6-987e-933ea75ca6db req-8e277f4c-30cb-4bb6-9723-f0edbdff3a5a service nova] Lock "09e003b7-3c4c-4b11-a52d-749acf709068-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1717.598194] env[67752]: DEBUG nova.compute.manager [req-bae7e9a4-513e-41e6-987e-933ea75ca6db req-8e277f4c-30cb-4bb6-9723-f0edbdff3a5a service nova] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] No waiting events found dispatching network-vif-plugged-0be69bd5-bbce-42e8-a36f-d8bf7ee56d35 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1717.598428] env[67752]: WARNING nova.compute.manager [req-bae7e9a4-513e-41e6-987e-933ea75ca6db req-8e277f4c-30cb-4bb6-9723-f0edbdff3a5a service nova] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Received unexpected event network-vif-plugged-0be69bd5-bbce-42e8-a36f-d8bf7ee56d35 for instance with vm_state building and task_state spawning. [ 1717.598649] env[67752]: DEBUG nova.compute.manager [req-bae7e9a4-513e-41e6-987e-933ea75ca6db req-8e277f4c-30cb-4bb6-9723-f0edbdff3a5a service nova] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Received event network-changed-0be69bd5-bbce-42e8-a36f-d8bf7ee56d35 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1717.598863] env[67752]: DEBUG nova.compute.manager [req-bae7e9a4-513e-41e6-987e-933ea75ca6db req-8e277f4c-30cb-4bb6-9723-f0edbdff3a5a service nova] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Refreshing instance network info cache due to event network-changed-0be69bd5-bbce-42e8-a36f-d8bf7ee56d35. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1717.599257] env[67752]: DEBUG oslo_concurrency.lockutils [req-bae7e9a4-513e-41e6-987e-933ea75ca6db req-8e277f4c-30cb-4bb6-9723-f0edbdff3a5a service nova] Acquiring lock "refresh_cache-09e003b7-3c4c-4b11-a52d-749acf709068" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1717.705555] env[67752]: DEBUG nova.network.neutron [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Updating instance_info_cache with network_info: [{"id": "0be69bd5-bbce-42e8-a36f-d8bf7ee56d35", "address": "fa:16:3e:c1:c1:f4", "network": {"id": "0fbb58d7-f336-4105-b40d-14fe23cb5231", "bridge": "br-int", "label": "tempest-ServersTestJSON-455316140-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "17776dc4e25145179ebfbdfd4a9639f8", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "92cdccfd-4b10-4024-b724-5f22792dd4de", "external-id": "nsx-vlan-transportzone-902", "segmentation_id": 902, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap0be69bd5-bb", "ovs_interfaceid": "0be69bd5-bbce-42e8-a36f-d8bf7ee56d35", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1717.716586] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Releasing lock "refresh_cache-09e003b7-3c4c-4b11-a52d-749acf709068" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1717.716865] env[67752]: DEBUG nova.compute.manager [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Instance network_info: |[{"id": "0be69bd5-bbce-42e8-a36f-d8bf7ee56d35", "address": "fa:16:3e:c1:c1:f4", "network": {"id": "0fbb58d7-f336-4105-b40d-14fe23cb5231", "bridge": "br-int", "label": "tempest-ServersTestJSON-455316140-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "17776dc4e25145179ebfbdfd4a9639f8", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "92cdccfd-4b10-4024-b724-5f22792dd4de", "external-id": "nsx-vlan-transportzone-902", "segmentation_id": 902, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap0be69bd5-bb", "ovs_interfaceid": "0be69bd5-bbce-42e8-a36f-d8bf7ee56d35", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1717.717174] env[67752]: DEBUG oslo_concurrency.lockutils [req-bae7e9a4-513e-41e6-987e-933ea75ca6db req-8e277f4c-30cb-4bb6-9723-f0edbdff3a5a service nova] Acquired lock "refresh_cache-09e003b7-3c4c-4b11-a52d-749acf709068" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1717.717415] env[67752]: DEBUG nova.network.neutron [req-bae7e9a4-513e-41e6-987e-933ea75ca6db req-8e277f4c-30cb-4bb6-9723-f0edbdff3a5a service nova] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Refreshing network info cache for port 0be69bd5-bbce-42e8-a36f-d8bf7ee56d35 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1717.718453] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:c1:c1:f4', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '92cdccfd-4b10-4024-b724-5f22792dd4de', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '0be69bd5-bbce-42e8-a36f-d8bf7ee56d35', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1717.725879] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Creating folder: Project (17776dc4e25145179ebfbdfd4a9639f8). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1717.727127] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-092cfe8d-e214-4e38-8a6c-d0e1899dacdd {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1717.740478] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Created folder: Project (17776dc4e25145179ebfbdfd4a9639f8) in parent group-v639722. [ 1717.740478] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Creating folder: Instances. Parent ref: group-v639827. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1717.740478] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-447e716d-69a5-4442-9ebe-c713c6b28691 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1717.748571] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Created folder: Instances in parent group-v639827. [ 1717.748780] env[67752]: DEBUG oslo.service.loopingcall [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1717.748984] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1717.749314] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-60b5b79e-7fa8-4e0d-ad7e-457e5bdda8a8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1717.770974] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1717.770974] env[67752]: value = "task-3199811" [ 1717.770974] env[67752]: _type = "Task" [ 1717.770974] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1717.778227] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199811, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1718.064080] env[67752]: DEBUG nova.network.neutron [req-bae7e9a4-513e-41e6-987e-933ea75ca6db req-8e277f4c-30cb-4bb6-9723-f0edbdff3a5a service nova] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Updated VIF entry in instance network info cache for port 0be69bd5-bbce-42e8-a36f-d8bf7ee56d35. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1718.064547] env[67752]: DEBUG nova.network.neutron [req-bae7e9a4-513e-41e6-987e-933ea75ca6db req-8e277f4c-30cb-4bb6-9723-f0edbdff3a5a service nova] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Updating instance_info_cache with network_info: [{"id": "0be69bd5-bbce-42e8-a36f-d8bf7ee56d35", "address": "fa:16:3e:c1:c1:f4", "network": {"id": "0fbb58d7-f336-4105-b40d-14fe23cb5231", "bridge": "br-int", "label": "tempest-ServersTestJSON-455316140-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "17776dc4e25145179ebfbdfd4a9639f8", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "92cdccfd-4b10-4024-b724-5f22792dd4de", "external-id": "nsx-vlan-transportzone-902", "segmentation_id": 902, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap0be69bd5-bb", "ovs_interfaceid": "0be69bd5-bbce-42e8-a36f-d8bf7ee56d35", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1718.074122] env[67752]: DEBUG oslo_concurrency.lockutils [req-bae7e9a4-513e-41e6-987e-933ea75ca6db req-8e277f4c-30cb-4bb6-9723-f0edbdff3a5a service nova] Releasing lock "refresh_cache-09e003b7-3c4c-4b11-a52d-749acf709068" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1718.281236] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199811, 'name': CreateVM_Task} progress is 99%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1718.781219] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199811, 'name': CreateVM_Task, 'duration_secs': 0.522542} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1718.781607] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1718.782037] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1718.782210] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1718.782546] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1718.782787] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b303f245-dee6-4bc3-be25-d8b02dfa64c4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1718.787133] env[67752]: DEBUG oslo_vmware.api [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Waiting for the task: (returnval){ [ 1718.787133] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5209f522-d2a8-05c8-f830-efdadda4c12d" [ 1718.787133] env[67752]: _type = "Task" [ 1718.787133] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1718.794177] env[67752]: DEBUG oslo_vmware.api [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5209f522-d2a8-05c8-f830-efdadda4c12d, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1719.298607] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1719.298607] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1719.298607] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1736.270677] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquiring lock "f2509f7b-1f6b-4942-b591-b7c691fafb26" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1736.271052] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "f2509f7b-1f6b-4942-b591-b7c691fafb26" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1741.634803] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1741.635169] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Cleaning up deleted instances with incomplete migration {{(pid=67752) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11257}} [ 1742.639752] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1747.635673] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1747.636110] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1747.636110] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Cleaning up deleted instances {{(pid=67752) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11219}} [ 1747.647080] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] There are 0 instances to clean {{(pid=67752) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11228}} [ 1749.646530] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1749.646825] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1749.646948] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1749.647138] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 1750.635583] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1750.635774] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 1750.635898] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 1750.657614] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1750.657929] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1750.657929] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1750.658037] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1750.658163] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1750.658289] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1750.658411] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1750.658531] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1750.658651] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1750.658771] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1750.658971] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 1750.659490] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1752.654848] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1755.634650] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1755.635108] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1755.647597] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1755.647809] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1755.648014] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1755.648147] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1755.649239] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1f2d4373-caf8-4b5f-ad50-3f90b31f3cc3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1755.657663] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8901f3bb-00c4-462e-8452-96354c0a38a2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1755.671075] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b3b9f659-c7aa-4c48-b5a7-d1901ab4e263 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1755.676910] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-68249a45-6f30-4384-af1d-bc8a7554c358 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1755.704773] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181026MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1755.704912] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1755.705111] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1755.845978] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 9432cd04-c2df-428b-b31a-93d3ec0c9d1f actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1755.846168] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 90e58033-eacd-433a-8d25-40f7d8403588 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1755.846300] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 81738fe6-6d50-4dfe-ae86-4c8771837168 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1755.846428] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e4ebb0f0-ab58-491f-bc4a-7548bbe415d8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1755.846550] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c1975c17-15a5-4751-bf7f-d5b7e0d2fea1 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1755.846670] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1755.846786] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 7557e3a1-b298-401f-8320-5a48ddda3298 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1755.846928] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 34c75d7e-e63e-4eff-afad-0edd61e023fc actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1755.847086] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d52496eb-f8e3-4d86-8454-f56e9097777e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1755.847207] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 09e003b7-3c4c-4b11-a52d-749acf709068 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1755.858297] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fea66a16-92c3-4c7a-8bff-c6eb59270158 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1755.868683] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f2509f7b-1f6b-4942-b591-b7c691fafb26 has been scheduled to this compute host, the scheduler has made an allocation against this compute node but the instance has yet to start. Skipping heal of allocation: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1764}} [ 1755.868984] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1755.869345] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1756.004642] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b73ffe8d-941a-4ecf-8270-3df763b4ea23 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1756.013365] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5da2bbd3-dc5d-42a0-93de-abde596b88d0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1756.041980] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-094a93d4-4d05-4ca2-84b9-07f58d5a7484 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1756.048835] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-38b50e6a-f7c7-4644-bc52-5f3988a6903d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1756.061989] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1756.070033] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1756.083719] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1756.083831] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.379s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1762.323880] env[67752]: WARNING oslo_vmware.rw_handles [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1762.323880] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1762.323880] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1762.323880] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1762.323880] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1762.323880] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1762.323880] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1762.323880] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1762.323880] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1762.323880] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1762.323880] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1762.323880] env[67752]: ERROR oslo_vmware.rw_handles [ 1762.324639] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/37db4d0f-58c0-402c-aa44-9ee10fa51a33/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1762.326882] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1762.327209] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Copying Virtual Disk [datastore2] vmware_temp/37db4d0f-58c0-402c-aa44-9ee10fa51a33/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/37db4d0f-58c0-402c-aa44-9ee10fa51a33/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1762.327535] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-19d0869a-7413-4ef6-bced-98cae90543d6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1762.336237] env[67752]: DEBUG oslo_vmware.api [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Waiting for the task: (returnval){ [ 1762.336237] env[67752]: value = "task-3199812" [ 1762.336237] env[67752]: _type = "Task" [ 1762.336237] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1762.343699] env[67752]: DEBUG oslo_vmware.api [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Task: {'id': task-3199812, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1762.847014] env[67752]: DEBUG oslo_vmware.exceptions [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1762.847329] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1762.847901] env[67752]: ERROR nova.compute.manager [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1762.847901] env[67752]: Faults: ['InvalidArgument'] [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Traceback (most recent call last): [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] yield resources [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] self.driver.spawn(context, instance, image_meta, [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] self._fetch_image_if_missing(context, vi) [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] image_cache(vi, tmp_image_ds_loc) [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] vm_util.copy_virtual_disk( [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] session._wait_for_task(vmdk_copy_task) [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] return self.wait_for_task(task_ref) [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] return evt.wait() [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] result = hub.switch() [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] return self.greenlet.switch() [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] self.f(*self.args, **self.kw) [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] raise exceptions.translate_fault(task_info.error) [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Faults: ['InvalidArgument'] [ 1762.847901] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] [ 1762.848895] env[67752]: INFO nova.compute.manager [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Terminating instance [ 1762.850451] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1762.850617] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1762.850856] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-cccb02c1-fe94-43f6-ba10-509db2a1d8fc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1762.853160] env[67752]: DEBUG nova.compute.manager [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1762.853356] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1762.854095] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7dde278-c775-4560-860f-a8f3fbf13208 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1762.861038] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1762.861263] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-5af73c23-0924-49a1-9a22-f2fecd642f79 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1762.863388] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1762.863562] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1762.864470] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-963ad923-72e8-4175-b5b1-fbbc7c003269 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1762.869065] env[67752]: DEBUG oslo_vmware.api [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Waiting for the task: (returnval){ [ 1762.869065] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]522bea74-a46f-48f2-6602-2332b8f75aeb" [ 1762.869065] env[67752]: _type = "Task" [ 1762.869065] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1762.877365] env[67752]: DEBUG oslo_vmware.api [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]522bea74-a46f-48f2-6602-2332b8f75aeb, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1762.934171] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1762.934171] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1762.934469] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Deleting the datastore file [datastore2] 9432cd04-c2df-428b-b31a-93d3ec0c9d1f {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1762.934565] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-4a6c0a45-2b06-42dd-97d7-a25667e9024c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1762.941987] env[67752]: DEBUG oslo_vmware.api [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Waiting for the task: (returnval){ [ 1762.941987] env[67752]: value = "task-3199814" [ 1762.941987] env[67752]: _type = "Task" [ 1762.941987] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1762.949457] env[67752]: DEBUG oslo_vmware.api [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Task: {'id': task-3199814, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1763.379241] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1763.379544] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Creating directory with path [datastore2] vmware_temp/d53a2c29-903a-4b6a-8cab-f019f4990b95/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1763.379741] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-85a86e0f-1907-4cc4-855d-08bea7f0113a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1763.390762] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Created directory with path [datastore2] vmware_temp/d53a2c29-903a-4b6a-8cab-f019f4990b95/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1763.390940] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Fetch image to [datastore2] vmware_temp/d53a2c29-903a-4b6a-8cab-f019f4990b95/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1763.391119] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/d53a2c29-903a-4b6a-8cab-f019f4990b95/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1763.391802] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bdb8a7f0-6d9e-46b0-a854-78afc575f23d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1763.397899] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2f295970-3235-4297-8acc-fe6490d34190 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1763.406391] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-db8074be-cb7d-482c-a0ad-612c6f201503 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1763.436409] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-19ef1090-b87e-4d57-8cec-9779d33f9bb7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1763.441878] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-ff0eb6da-5b8b-4417-aea2-41c74270a5a4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1763.451035] env[67752]: DEBUG oslo_vmware.api [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Task: {'id': task-3199814, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.081372} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1763.451320] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1763.451527] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1763.451703] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1763.451881] env[67752]: INFO nova.compute.manager [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1763.454105] env[67752]: DEBUG nova.compute.claims [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1763.454763] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1763.454763] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1763.466898] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1763.517094] env[67752]: DEBUG oslo_vmware.rw_handles [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/d53a2c29-903a-4b6a-8cab-f019f4990b95/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1763.579425] env[67752]: DEBUG oslo_vmware.rw_handles [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1763.579608] env[67752]: DEBUG oslo_vmware.rw_handles [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Closing write handle for https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/d53a2c29-903a-4b6a-8cab-f019f4990b95/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1763.634750] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1763.689782] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a8bc228b-0a50-4b31-8c4c-2ad85a295f08 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1763.696952] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2afad279-3f48-4d29-bad3-7ab767ceace5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1763.727087] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a6fe55ae-e970-4e72-a69b-6941a0b5423f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1763.733673] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54d43c47-0181-4a67-9221-153e0d8a261f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1763.746130] env[67752]: DEBUG nova.compute.provider_tree [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1763.755831] env[67752]: DEBUG nova.scheduler.client.report [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1763.769750] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.315s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1763.770777] env[67752]: ERROR nova.compute.manager [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1763.770777] env[67752]: Faults: ['InvalidArgument'] [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Traceback (most recent call last): [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] self.driver.spawn(context, instance, image_meta, [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] self._fetch_image_if_missing(context, vi) [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] image_cache(vi, tmp_image_ds_loc) [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] vm_util.copy_virtual_disk( [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] session._wait_for_task(vmdk_copy_task) [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] return self.wait_for_task(task_ref) [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] return evt.wait() [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] result = hub.switch() [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] return self.greenlet.switch() [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] self.f(*self.args, **self.kw) [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] raise exceptions.translate_fault(task_info.error) [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Faults: ['InvalidArgument'] [ 1763.770777] env[67752]: ERROR nova.compute.manager [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] [ 1763.770777] env[67752]: DEBUG nova.compute.utils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1763.772280] env[67752]: DEBUG nova.compute.manager [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Build of instance 9432cd04-c2df-428b-b31a-93d3ec0c9d1f was re-scheduled: A specified parameter was not correct: fileType [ 1763.772280] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1763.772666] env[67752]: DEBUG nova.compute.manager [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1763.772843] env[67752]: DEBUG nova.compute.manager [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1763.773026] env[67752]: DEBUG nova.compute.manager [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1763.773197] env[67752]: DEBUG nova.network.neutron [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1764.214884] env[67752]: DEBUG nova.network.neutron [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1764.228029] env[67752]: INFO nova.compute.manager [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Took 0.45 seconds to deallocate network for instance. [ 1764.319567] env[67752]: INFO nova.scheduler.client.report [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Deleted allocations for instance 9432cd04-c2df-428b-b31a-93d3ec0c9d1f [ 1764.342014] env[67752]: DEBUG oslo_concurrency.lockutils [None req-8addf2bd-7bb2-4500-81c1-5d6e540b5e02 tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Lock "9432cd04-c2df-428b-b31a-93d3ec0c9d1f" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 672.744s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1764.343141] env[67752]: DEBUG oslo_concurrency.lockutils [None req-79213572-3c2a-410f-a3f0-655d520da73e tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Lock "9432cd04-c2df-428b-b31a-93d3ec0c9d1f" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 476.282s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1764.343368] env[67752]: DEBUG oslo_concurrency.lockutils [None req-79213572-3c2a-410f-a3f0-655d520da73e tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Acquiring lock "9432cd04-c2df-428b-b31a-93d3ec0c9d1f-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1764.343577] env[67752]: DEBUG oslo_concurrency.lockutils [None req-79213572-3c2a-410f-a3f0-655d520da73e tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Lock "9432cd04-c2df-428b-b31a-93d3ec0c9d1f-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1764.343746] env[67752]: DEBUG oslo_concurrency.lockutils [None req-79213572-3c2a-410f-a3f0-655d520da73e tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Lock "9432cd04-c2df-428b-b31a-93d3ec0c9d1f-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1764.345916] env[67752]: INFO nova.compute.manager [None req-79213572-3c2a-410f-a3f0-655d520da73e tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Terminating instance [ 1764.347740] env[67752]: DEBUG nova.compute.manager [None req-79213572-3c2a-410f-a3f0-655d520da73e tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1764.347934] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-79213572-3c2a-410f-a3f0-655d520da73e tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1764.348397] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-81f81537-6c72-4c04-919e-eae842c9ed59 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1764.353266] env[67752]: DEBUG nova.compute.manager [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1764.360089] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9987e5e3-c860-4781-b25e-e5150fc161ca {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1764.389181] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-79213572-3c2a-410f-a3f0-655d520da73e tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 9432cd04-c2df-428b-b31a-93d3ec0c9d1f could not be found. [ 1764.389507] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-79213572-3c2a-410f-a3f0-655d520da73e tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1764.389604] env[67752]: INFO nova.compute.manager [None req-79213572-3c2a-410f-a3f0-655d520da73e tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1764.389859] env[67752]: DEBUG oslo.service.loopingcall [None req-79213572-3c2a-410f-a3f0-655d520da73e tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1764.390989] env[67752]: DEBUG nova.compute.manager [-] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1764.390989] env[67752]: DEBUG nova.network.neutron [-] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1764.406749] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1764.406975] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1764.408458] env[67752]: INFO nova.compute.claims [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1764.436857] env[67752]: DEBUG nova.network.neutron [-] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1764.445447] env[67752]: INFO nova.compute.manager [-] [instance: 9432cd04-c2df-428b-b31a-93d3ec0c9d1f] Took 0.05 seconds to deallocate network for instance. [ 1764.540820] env[67752]: DEBUG oslo_concurrency.lockutils [None req-79213572-3c2a-410f-a3f0-655d520da73e tempest-AttachInterfacesV270Test-904860710 tempest-AttachInterfacesV270Test-904860710-project-member] Lock "9432cd04-c2df-428b-b31a-93d3ec0c9d1f" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.198s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1764.606292] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-65ff2fa7-a1e8-4fb9-b360-cb3230fe676b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1764.613723] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5aaf5636-c8d1-4ef0-9eaa-e67b8cfc079a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1764.643791] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-15ab8c07-cb68-452a-9e40-dd0199924d49 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1764.650379] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b83cbcb8-7907-44e8-8519-cc9d48269aed {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1764.663919] env[67752]: DEBUG nova.compute.provider_tree [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1764.672630] env[67752]: DEBUG nova.scheduler.client.report [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1764.684819] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.278s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1764.685282] env[67752]: DEBUG nova.compute.manager [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1764.718603] env[67752]: DEBUG nova.compute.utils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1764.720146] env[67752]: DEBUG nova.compute.manager [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1764.720349] env[67752]: DEBUG nova.network.neutron [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1764.729284] env[67752]: DEBUG nova.compute.manager [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1764.784686] env[67752]: DEBUG nova.policy [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': 'ec812057c5984ac9b1651df9bba8a354', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '09d8d9a5042e4746926c2bc7d7236b49', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1764.793090] env[67752]: DEBUG nova.compute.manager [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1764.818125] env[67752]: DEBUG nova.virt.hardware [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1764.818372] env[67752]: DEBUG nova.virt.hardware [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1764.818531] env[67752]: DEBUG nova.virt.hardware [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1764.818715] env[67752]: DEBUG nova.virt.hardware [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1764.818864] env[67752]: DEBUG nova.virt.hardware [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1764.819022] env[67752]: DEBUG nova.virt.hardware [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1764.819266] env[67752]: DEBUG nova.virt.hardware [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1764.819436] env[67752]: DEBUG nova.virt.hardware [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1764.819607] env[67752]: DEBUG nova.virt.hardware [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1764.819770] env[67752]: DEBUG nova.virt.hardware [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1764.819942] env[67752]: DEBUG nova.virt.hardware [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1764.820802] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2ec0cf46-9a43-4df0-b249-7e2196fb19ba {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1764.828543] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b6c77773-2433-48ed-be35-e6d54c6688d4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1765.438730] env[67752]: DEBUG nova.network.neutron [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Successfully created port: 3ff78b72-76b1-469f-a894-1daafcf99b1b {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1766.249553] env[67752]: DEBUG nova.network.neutron [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Successfully updated port: 3ff78b72-76b1-469f-a894-1daafcf99b1b {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1766.257088] env[67752]: DEBUG nova.compute.manager [req-57acd297-5581-46bd-b4cb-296a4225fb61 req-ee9e788c-9356-4687-b3a2-63a60a803cb2 service nova] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Received event network-vif-plugged-3ff78b72-76b1-469f-a894-1daafcf99b1b {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1766.258088] env[67752]: DEBUG oslo_concurrency.lockutils [req-57acd297-5581-46bd-b4cb-296a4225fb61 req-ee9e788c-9356-4687-b3a2-63a60a803cb2 service nova] Acquiring lock "fea66a16-92c3-4c7a-8bff-c6eb59270158-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1766.258088] env[67752]: DEBUG oslo_concurrency.lockutils [req-57acd297-5581-46bd-b4cb-296a4225fb61 req-ee9e788c-9356-4687-b3a2-63a60a803cb2 service nova] Lock "fea66a16-92c3-4c7a-8bff-c6eb59270158-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1766.258088] env[67752]: DEBUG oslo_concurrency.lockutils [req-57acd297-5581-46bd-b4cb-296a4225fb61 req-ee9e788c-9356-4687-b3a2-63a60a803cb2 service nova] Lock "fea66a16-92c3-4c7a-8bff-c6eb59270158-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1766.258266] env[67752]: DEBUG nova.compute.manager [req-57acd297-5581-46bd-b4cb-296a4225fb61 req-ee9e788c-9356-4687-b3a2-63a60a803cb2 service nova] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] No waiting events found dispatching network-vif-plugged-3ff78b72-76b1-469f-a894-1daafcf99b1b {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1766.258570] env[67752]: WARNING nova.compute.manager [req-57acd297-5581-46bd-b4cb-296a4225fb61 req-ee9e788c-9356-4687-b3a2-63a60a803cb2 service nova] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Received unexpected event network-vif-plugged-3ff78b72-76b1-469f-a894-1daafcf99b1b for instance with vm_state building and task_state spawning. [ 1766.259987] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Acquiring lock "refresh_cache-fea66a16-92c3-4c7a-8bff-c6eb59270158" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1766.260267] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Acquired lock "refresh_cache-fea66a16-92c3-4c7a-8bff-c6eb59270158" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1766.260524] env[67752]: DEBUG nova.network.neutron [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1766.306238] env[67752]: DEBUG nova.network.neutron [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1766.559449] env[67752]: DEBUG nova.network.neutron [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Updating instance_info_cache with network_info: [{"id": "3ff78b72-76b1-469f-a894-1daafcf99b1b", "address": "fa:16:3e:25:eb:73", "network": {"id": "8d607924-af5d-42a4-9f3f-d6ed51da594d", "bridge": "br-int", "label": "tempest-AttachInterfacesTestJSON-1105783943-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "09d8d9a5042e4746926c2bc7d7236b49", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "9f52a458-d157-48a3-b4e2-b8cc0779afe2", "external-id": "nsx-vlan-transportzone-403", "segmentation_id": 403, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap3ff78b72-76", "ovs_interfaceid": "3ff78b72-76b1-469f-a894-1daafcf99b1b", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1766.573258] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Releasing lock "refresh_cache-fea66a16-92c3-4c7a-8bff-c6eb59270158" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1766.573489] env[67752]: DEBUG nova.compute.manager [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Instance network_info: |[{"id": "3ff78b72-76b1-469f-a894-1daafcf99b1b", "address": "fa:16:3e:25:eb:73", "network": {"id": "8d607924-af5d-42a4-9f3f-d6ed51da594d", "bridge": "br-int", "label": "tempest-AttachInterfacesTestJSON-1105783943-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "09d8d9a5042e4746926c2bc7d7236b49", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "9f52a458-d157-48a3-b4e2-b8cc0779afe2", "external-id": "nsx-vlan-transportzone-403", "segmentation_id": 403, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap3ff78b72-76", "ovs_interfaceid": "3ff78b72-76b1-469f-a894-1daafcf99b1b", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1766.573902] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:25:eb:73', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '9f52a458-d157-48a3-b4e2-b8cc0779afe2', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '3ff78b72-76b1-469f-a894-1daafcf99b1b', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1766.581660] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Creating folder: Project (09d8d9a5042e4746926c2bc7d7236b49). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1766.582175] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-de07fb76-cfd4-49c2-a53e-bdf4691b23cf {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1766.593335] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Created folder: Project (09d8d9a5042e4746926c2bc7d7236b49) in parent group-v639722. [ 1766.593515] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Creating folder: Instances. Parent ref: group-v639830. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 1766.593724] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-f63bcb4c-28bb-4f21-9a54-e669967a4383 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1766.601891] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Created folder: Instances in parent group-v639830. [ 1766.602124] env[67752]: DEBUG oslo.service.loopingcall [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1766.602299] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1766.602511] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-81b3544f-47a6-45c4-bb9a-94f28b4aaf18 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1766.620050] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1766.620050] env[67752]: value = "task-3199817" [ 1766.620050] env[67752]: _type = "Task" [ 1766.620050] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1766.627021] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199817, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1767.130275] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199817, 'name': CreateVM_Task, 'duration_secs': 0.315693} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1767.130467] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1767.131136] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1767.131307] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1767.131663] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1767.131908] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-bad4ca22-660d-4a23-9dc5-190a363a97a0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1767.136054] env[67752]: DEBUG oslo_vmware.api [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Waiting for the task: (returnval){ [ 1767.136054] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]521aaf21-b5b9-b902-80d1-1bb1216b9429" [ 1767.136054] env[67752]: _type = "Task" [ 1767.136054] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1767.144500] env[67752]: DEBUG oslo_vmware.api [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]521aaf21-b5b9-b902-80d1-1bb1216b9429, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1767.646472] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1767.646758] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1767.647019] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1768.281686] env[67752]: DEBUG nova.compute.manager [req-451d57c4-ace5-426a-858f-137b20f24c0f req-049b161d-a3fd-4b13-bf36-1ca2f6e8139f service nova] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Received event network-changed-3ff78b72-76b1-469f-a894-1daafcf99b1b {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1768.281911] env[67752]: DEBUG nova.compute.manager [req-451d57c4-ace5-426a-858f-137b20f24c0f req-049b161d-a3fd-4b13-bf36-1ca2f6e8139f service nova] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Refreshing instance network info cache due to event network-changed-3ff78b72-76b1-469f-a894-1daafcf99b1b. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1768.282140] env[67752]: DEBUG oslo_concurrency.lockutils [req-451d57c4-ace5-426a-858f-137b20f24c0f req-049b161d-a3fd-4b13-bf36-1ca2f6e8139f service nova] Acquiring lock "refresh_cache-fea66a16-92c3-4c7a-8bff-c6eb59270158" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1768.282290] env[67752]: DEBUG oslo_concurrency.lockutils [req-451d57c4-ace5-426a-858f-137b20f24c0f req-049b161d-a3fd-4b13-bf36-1ca2f6e8139f service nova] Acquired lock "refresh_cache-fea66a16-92c3-4c7a-8bff-c6eb59270158" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1768.282454] env[67752]: DEBUG nova.network.neutron [req-451d57c4-ace5-426a-858f-137b20f24c0f req-049b161d-a3fd-4b13-bf36-1ca2f6e8139f service nova] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Refreshing network info cache for port 3ff78b72-76b1-469f-a894-1daafcf99b1b {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1768.533180] env[67752]: DEBUG nova.network.neutron [req-451d57c4-ace5-426a-858f-137b20f24c0f req-049b161d-a3fd-4b13-bf36-1ca2f6e8139f service nova] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Updated VIF entry in instance network info cache for port 3ff78b72-76b1-469f-a894-1daafcf99b1b. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1768.533653] env[67752]: DEBUG nova.network.neutron [req-451d57c4-ace5-426a-858f-137b20f24c0f req-049b161d-a3fd-4b13-bf36-1ca2f6e8139f service nova] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Updating instance_info_cache with network_info: [{"id": "3ff78b72-76b1-469f-a894-1daafcf99b1b", "address": "fa:16:3e:25:eb:73", "network": {"id": "8d607924-af5d-42a4-9f3f-d6ed51da594d", "bridge": "br-int", "label": "tempest-AttachInterfacesTestJSON-1105783943-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "09d8d9a5042e4746926c2bc7d7236b49", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "9f52a458-d157-48a3-b4e2-b8cc0779afe2", "external-id": "nsx-vlan-transportzone-403", "segmentation_id": 403, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap3ff78b72-76", "ovs_interfaceid": "3ff78b72-76b1-469f-a894-1daafcf99b1b", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1768.544639] env[67752]: DEBUG oslo_concurrency.lockutils [req-451d57c4-ace5-426a-858f-137b20f24c0f req-049b161d-a3fd-4b13-bf36-1ca2f6e8139f service nova] Releasing lock "refresh_cache-fea66a16-92c3-4c7a-8bff-c6eb59270158" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1791.863179] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._sync_power_states {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1791.886502] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Getting list of instances from cluster (obj){ [ 1791.886502] env[67752]: value = "domain-c8" [ 1791.886502] env[67752]: _type = "ClusterComputeResource" [ 1791.886502] env[67752]: } {{(pid=67752) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 1791.888309] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-25559657-eb03-4161-8749-230c3fa137c6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1791.905873] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Got total of 10 instances {{(pid=67752) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 1791.906069] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid 90e58033-eacd-433a-8d25-40f7d8403588 {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1791.906270] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid 81738fe6-6d50-4dfe-ae86-4c8771837168 {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1791.906429] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid e4ebb0f0-ab58-491f-bc4a-7548bbe415d8 {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1791.906586] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid c1975c17-15a5-4751-bf7f-d5b7e0d2fea1 {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1791.906738] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010 {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1791.906891] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid 7557e3a1-b298-401f-8320-5a48ddda3298 {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1791.907059] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid 34c75d7e-e63e-4eff-afad-0edd61e023fc {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1791.907215] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid d52496eb-f8e3-4d86-8454-f56e9097777e {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1791.907360] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid 09e003b7-3c4c-4b11-a52d-749acf709068 {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1791.907505] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid fea66a16-92c3-4c7a-8bff-c6eb59270158 {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 1791.907858] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "90e58033-eacd-433a-8d25-40f7d8403588" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1791.908130] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "81738fe6-6d50-4dfe-ae86-4c8771837168" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1791.908337] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "e4ebb0f0-ab58-491f-bc4a-7548bbe415d8" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1791.908534] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "c1975c17-15a5-4751-bf7f-d5b7e0d2fea1" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1791.908727] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "0f3f8622-4b01-47d3-bbbb-dcd88c7ca010" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1791.908920] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "7557e3a1-b298-401f-8320-5a48ddda3298" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1791.909145] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "34c75d7e-e63e-4eff-afad-0edd61e023fc" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1791.909347] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "d52496eb-f8e3-4d86-8454-f56e9097777e" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1791.909541] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "09e003b7-3c4c-4b11-a52d-749acf709068" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1791.909733] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "fea66a16-92c3-4c7a-8bff-c6eb59270158" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1804.677344] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1808.635684] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1808.876118] env[67752]: WARNING oslo_vmware.rw_handles [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1808.876118] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1808.876118] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1808.876118] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1808.876118] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1808.876118] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1808.876118] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1808.876118] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1808.876118] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1808.876118] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1808.876118] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1808.876118] env[67752]: ERROR oslo_vmware.rw_handles [ 1808.876621] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/d53a2c29-903a-4b6a-8cab-f019f4990b95/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1808.878894] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1808.879164] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Copying Virtual Disk [datastore2] vmware_temp/d53a2c29-903a-4b6a-8cab-f019f4990b95/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/d53a2c29-903a-4b6a-8cab-f019f4990b95/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1808.879448] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-e752fdd2-a4fb-4aa9-98e0-22180c2d2c9d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1808.887037] env[67752]: DEBUG oslo_vmware.api [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Waiting for the task: (returnval){ [ 1808.887037] env[67752]: value = "task-3199818" [ 1808.887037] env[67752]: _type = "Task" [ 1808.887037] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1808.895128] env[67752]: DEBUG oslo_vmware.api [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Task: {'id': task-3199818, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1809.397455] env[67752]: DEBUG oslo_vmware.exceptions [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1809.397744] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1809.398353] env[67752]: ERROR nova.compute.manager [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1809.398353] env[67752]: Faults: ['InvalidArgument'] [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Traceback (most recent call last): [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] yield resources [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] self.driver.spawn(context, instance, image_meta, [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] self._fetch_image_if_missing(context, vi) [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] image_cache(vi, tmp_image_ds_loc) [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] vm_util.copy_virtual_disk( [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] session._wait_for_task(vmdk_copy_task) [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] return self.wait_for_task(task_ref) [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] return evt.wait() [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] result = hub.switch() [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] return self.greenlet.switch() [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] self.f(*self.args, **self.kw) [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] raise exceptions.translate_fault(task_info.error) [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Faults: ['InvalidArgument'] [ 1809.398353] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] [ 1809.399504] env[67752]: INFO nova.compute.manager [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Terminating instance [ 1809.400436] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1809.400658] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1809.401323] env[67752]: DEBUG nova.compute.manager [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1809.401523] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1809.401751] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-6834a0cb-22d4-4b95-ac45-a37245659666 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1809.405582] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-59402d3a-43dc-4306-a5eb-0ee6e71c96a6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1809.412191] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1809.412401] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-cd7b724f-7f85-4028-97a1-bd1f1c0136af {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1809.414446] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1809.414623] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1809.415532] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-3a9cdf5f-3a34-4608-9746-578870dbb5e6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1809.420094] env[67752]: DEBUG oslo_vmware.api [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Waiting for the task: (returnval){ [ 1809.420094] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52996fc9-0e3e-b036-40f2-7fae12ceabbc" [ 1809.420094] env[67752]: _type = "Task" [ 1809.420094] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1809.430170] env[67752]: DEBUG oslo_vmware.api [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52996fc9-0e3e-b036-40f2-7fae12ceabbc, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1809.482962] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1809.483258] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1809.483454] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Deleting the datastore file [datastore2] 90e58033-eacd-433a-8d25-40f7d8403588 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1809.483711] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-30796929-abe9-4c66-94af-5711a5347b52 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1809.489255] env[67752]: DEBUG oslo_vmware.api [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Waiting for the task: (returnval){ [ 1809.489255] env[67752]: value = "task-3199820" [ 1809.489255] env[67752]: _type = "Task" [ 1809.489255] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1809.496889] env[67752]: DEBUG oslo_vmware.api [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Task: {'id': task-3199820, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1809.634683] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1809.634919] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1809.635091] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1809.635251] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 1809.930151] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1809.930563] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Creating directory with path [datastore2] vmware_temp/69a675df-9c44-40fe-8d8a-7bb85a7a067e/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1809.930611] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-e164e464-9e60-4cde-a1e5-5d549178d940 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1809.941232] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Created directory with path [datastore2] vmware_temp/69a675df-9c44-40fe-8d8a-7bb85a7a067e/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1809.941420] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Fetch image to [datastore2] vmware_temp/69a675df-9c44-40fe-8d8a-7bb85a7a067e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1809.941581] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/69a675df-9c44-40fe-8d8a-7bb85a7a067e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1809.942279] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ca2565b8-6efe-4def-9d6c-b3b3ee41bd2a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1809.948701] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8eb6ffb7-4ab6-47f2-ae4b-bc25f521b186 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1809.957375] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6a3a1358-a872-41c0-a437-66dac0115aa0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1809.986820] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0e75785-5b82-468a-a719-1d973c056996 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1809.994393] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-a2c9ccf7-c277-4604-8367-44692c9d0d03 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1810.000131] env[67752]: DEBUG oslo_vmware.api [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Task: {'id': task-3199820, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.091635} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1810.000419] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1810.000556] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1810.000728] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1810.000898] env[67752]: INFO nova.compute.manager [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1810.003038] env[67752]: DEBUG nova.compute.claims [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1810.003231] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1810.003452] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1810.015996] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1810.071108] env[67752]: DEBUG oslo_vmware.rw_handles [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/69a675df-9c44-40fe-8d8a-7bb85a7a067e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1810.131246] env[67752]: DEBUG oslo_vmware.rw_handles [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1810.131429] env[67752]: DEBUG oslo_vmware.rw_handles [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Closing write handle for https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/69a675df-9c44-40fe-8d8a-7bb85a7a067e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1810.225181] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d4face6f-440c-4fbc-a78a-0a516cf8dbff {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1810.232353] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c844b91-6350-43ec-8a2b-e420b12f3f8b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1810.261311] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de5b7704-aabb-491a-b247-0c498c59233f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1810.267852] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3fde53fe-e38c-43bb-8492-8d37d5d4f933 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1810.280903] env[67752]: DEBUG nova.compute.provider_tree [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1810.289775] env[67752]: DEBUG nova.scheduler.client.report [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1810.304036] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.300s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1810.304456] env[67752]: ERROR nova.compute.manager [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1810.304456] env[67752]: Faults: ['InvalidArgument'] [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Traceback (most recent call last): [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] self.driver.spawn(context, instance, image_meta, [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] self._fetch_image_if_missing(context, vi) [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] image_cache(vi, tmp_image_ds_loc) [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] vm_util.copy_virtual_disk( [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] session._wait_for_task(vmdk_copy_task) [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] return self.wait_for_task(task_ref) [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] return evt.wait() [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] result = hub.switch() [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] return self.greenlet.switch() [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] self.f(*self.args, **self.kw) [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] raise exceptions.translate_fault(task_info.error) [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Faults: ['InvalidArgument'] [ 1810.304456] env[67752]: ERROR nova.compute.manager [instance: 90e58033-eacd-433a-8d25-40f7d8403588] [ 1810.305573] env[67752]: DEBUG nova.compute.utils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1810.306642] env[67752]: DEBUG nova.compute.manager [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Build of instance 90e58033-eacd-433a-8d25-40f7d8403588 was re-scheduled: A specified parameter was not correct: fileType [ 1810.306642] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1810.307035] env[67752]: DEBUG nova.compute.manager [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1810.307223] env[67752]: DEBUG nova.compute.manager [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1810.307400] env[67752]: DEBUG nova.compute.manager [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1810.307564] env[67752]: DEBUG nova.network.neutron [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1810.867289] env[67752]: DEBUG nova.network.neutron [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1810.880895] env[67752]: INFO nova.compute.manager [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Took 0.57 seconds to deallocate network for instance. [ 1810.981760] env[67752]: INFO nova.scheduler.client.report [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Deleted allocations for instance 90e58033-eacd-433a-8d25-40f7d8403588 [ 1811.003012] env[67752]: DEBUG oslo_concurrency.lockutils [None req-d1afff41-2213-460a-8cb6-41c27f8b8be8 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Lock "90e58033-eacd-433a-8d25-40f7d8403588" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 590.021s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1811.004531] env[67752]: DEBUG oslo_concurrency.lockutils [None req-747b37fc-be61-4aba-aa0f-8c0cde40f237 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Lock "90e58033-eacd-433a-8d25-40f7d8403588" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 393.760s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1811.004855] env[67752]: DEBUG oslo_concurrency.lockutils [None req-747b37fc-be61-4aba-aa0f-8c0cde40f237 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Acquiring lock "90e58033-eacd-433a-8d25-40f7d8403588-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1811.005104] env[67752]: DEBUG oslo_concurrency.lockutils [None req-747b37fc-be61-4aba-aa0f-8c0cde40f237 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Lock "90e58033-eacd-433a-8d25-40f7d8403588-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1811.005364] env[67752]: DEBUG oslo_concurrency.lockutils [None req-747b37fc-be61-4aba-aa0f-8c0cde40f237 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Lock "90e58033-eacd-433a-8d25-40f7d8403588-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1811.007498] env[67752]: INFO nova.compute.manager [None req-747b37fc-be61-4aba-aa0f-8c0cde40f237 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Terminating instance [ 1811.009188] env[67752]: DEBUG nova.compute.manager [None req-747b37fc-be61-4aba-aa0f-8c0cde40f237 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1811.009387] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-747b37fc-be61-4aba-aa0f-8c0cde40f237 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1811.009863] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-0972f843-ffb9-4c08-85eb-a430ac8baf70 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1811.014508] env[67752]: DEBUG nova.compute.manager [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1811.022017] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9dce7875-0a77-43e9-af3b-08907d76b8ec {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1811.052144] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-747b37fc-be61-4aba-aa0f-8c0cde40f237 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 90e58033-eacd-433a-8d25-40f7d8403588 could not be found. [ 1811.052438] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-747b37fc-be61-4aba-aa0f-8c0cde40f237 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1811.052630] env[67752]: INFO nova.compute.manager [None req-747b37fc-be61-4aba-aa0f-8c0cde40f237 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1811.052881] env[67752]: DEBUG oslo.service.loopingcall [None req-747b37fc-be61-4aba-aa0f-8c0cde40f237 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1811.057793] env[67752]: DEBUG nova.compute.manager [-] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1811.057913] env[67752]: DEBUG nova.network.neutron [-] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1811.069617] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1811.069859] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1811.071560] env[67752]: INFO nova.compute.claims [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1811.081462] env[67752]: DEBUG nova.network.neutron [-] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1811.096726] env[67752]: INFO nova.compute.manager [-] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] Took 0.04 seconds to deallocate network for instance. [ 1811.181422] env[67752]: DEBUG oslo_concurrency.lockutils [None req-747b37fc-be61-4aba-aa0f-8c0cde40f237 tempest-ServerAddressesNegativeTestJSON-353661731 tempest-ServerAddressesNegativeTestJSON-353661731-project-member] Lock "90e58033-eacd-433a-8d25-40f7d8403588" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.177s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1811.182325] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "90e58033-eacd-433a-8d25-40f7d8403588" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 19.274s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1811.182429] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 90e58033-eacd-433a-8d25-40f7d8403588] During sync_power_state the instance has a pending task (deleting). Skip. [ 1811.182721] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "90e58033-eacd-433a-8d25-40f7d8403588" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1811.247471] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-495370fd-e6b0-40e2-b7a4-37a64234056e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1811.255957] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-03fac551-a05a-4678-afe6-3428d096db38 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1811.285583] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6b9f6425-de3c-4100-806e-2291aee0213f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1811.292130] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d5c80773-c8ec-4fce-819e-cd3b48499ea1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1811.304588] env[67752]: DEBUG nova.compute.provider_tree [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1811.313408] env[67752]: DEBUG nova.scheduler.client.report [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1811.326120] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.256s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1811.326586] env[67752]: DEBUG nova.compute.manager [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1811.362237] env[67752]: DEBUG nova.compute.utils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1811.363838] env[67752]: DEBUG nova.compute.manager [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1811.363983] env[67752]: DEBUG nova.network.neutron [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1811.372722] env[67752]: DEBUG nova.compute.manager [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1811.423176] env[67752]: DEBUG nova.policy [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '639cb98e283640c3907a927d4c10c359', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'dc0e175514604a85bb7d9284ff4be671', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1811.436763] env[67752]: DEBUG nova.compute.manager [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1811.464780] env[67752]: DEBUG nova.virt.hardware [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1811.465030] env[67752]: DEBUG nova.virt.hardware [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1811.465179] env[67752]: DEBUG nova.virt.hardware [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1811.465424] env[67752]: DEBUG nova.virt.hardware [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1811.465498] env[67752]: DEBUG nova.virt.hardware [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1811.465822] env[67752]: DEBUG nova.virt.hardware [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1811.465883] env[67752]: DEBUG nova.virt.hardware [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1811.466046] env[67752]: DEBUG nova.virt.hardware [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1811.466246] env[67752]: DEBUG nova.virt.hardware [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1811.466420] env[67752]: DEBUG nova.virt.hardware [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1811.466621] env[67752]: DEBUG nova.virt.hardware [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1811.467525] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a2ae8e89-34e1-4273-9855-ec966080167c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1811.475764] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d4f993a2-4408-4c63-9149-3cacede601a9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1811.635807] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1812.059681] env[67752]: DEBUG nova.network.neutron [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Successfully created port: 1dbe4598-b834-41f2-a435-4cadf5bb8bda {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1812.635858] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1812.635858] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 1812.635858] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 1812.665039] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1812.665210] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1812.665340] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1812.665453] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1812.665581] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1812.665715] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1812.665834] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1812.665954] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1812.666148] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1812.666235] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1812.666363] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 1812.726989] env[67752]: DEBUG nova.network.neutron [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Successfully updated port: 1dbe4598-b834-41f2-a435-4cadf5bb8bda {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1812.739234] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquiring lock "refresh_cache-f2509f7b-1f6b-4942-b591-b7c691fafb26" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1812.739389] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquired lock "refresh_cache-f2509f7b-1f6b-4942-b591-b7c691fafb26" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1812.739544] env[67752]: DEBUG nova.network.neutron [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1812.826656] env[67752]: DEBUG nova.network.neutron [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1812.907668] env[67752]: DEBUG nova.compute.manager [req-54ede0e6-c120-4fc8-880a-64ebbcfc6cec req-82cdca3c-a6e2-482c-84c6-579eed1822bf service nova] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Received event network-vif-plugged-1dbe4598-b834-41f2-a435-4cadf5bb8bda {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1812.907952] env[67752]: DEBUG oslo_concurrency.lockutils [req-54ede0e6-c120-4fc8-880a-64ebbcfc6cec req-82cdca3c-a6e2-482c-84c6-579eed1822bf service nova] Acquiring lock "f2509f7b-1f6b-4942-b591-b7c691fafb26-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1812.908360] env[67752]: DEBUG oslo_concurrency.lockutils [req-54ede0e6-c120-4fc8-880a-64ebbcfc6cec req-82cdca3c-a6e2-482c-84c6-579eed1822bf service nova] Lock "f2509f7b-1f6b-4942-b591-b7c691fafb26-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1812.908746] env[67752]: DEBUG oslo_concurrency.lockutils [req-54ede0e6-c120-4fc8-880a-64ebbcfc6cec req-82cdca3c-a6e2-482c-84c6-579eed1822bf service nova] Lock "f2509f7b-1f6b-4942-b591-b7c691fafb26-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1812.908843] env[67752]: DEBUG nova.compute.manager [req-54ede0e6-c120-4fc8-880a-64ebbcfc6cec req-82cdca3c-a6e2-482c-84c6-579eed1822bf service nova] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] No waiting events found dispatching network-vif-plugged-1dbe4598-b834-41f2-a435-4cadf5bb8bda {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1812.909043] env[67752]: WARNING nova.compute.manager [req-54ede0e6-c120-4fc8-880a-64ebbcfc6cec req-82cdca3c-a6e2-482c-84c6-579eed1822bf service nova] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Received unexpected event network-vif-plugged-1dbe4598-b834-41f2-a435-4cadf5bb8bda for instance with vm_state building and task_state spawning. [ 1812.909260] env[67752]: DEBUG nova.compute.manager [req-54ede0e6-c120-4fc8-880a-64ebbcfc6cec req-82cdca3c-a6e2-482c-84c6-579eed1822bf service nova] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Received event network-changed-1dbe4598-b834-41f2-a435-4cadf5bb8bda {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1812.909426] env[67752]: DEBUG nova.compute.manager [req-54ede0e6-c120-4fc8-880a-64ebbcfc6cec req-82cdca3c-a6e2-482c-84c6-579eed1822bf service nova] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Refreshing instance network info cache due to event network-changed-1dbe4598-b834-41f2-a435-4cadf5bb8bda. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1812.909630] env[67752]: DEBUG oslo_concurrency.lockutils [req-54ede0e6-c120-4fc8-880a-64ebbcfc6cec req-82cdca3c-a6e2-482c-84c6-579eed1822bf service nova] Acquiring lock "refresh_cache-f2509f7b-1f6b-4942-b591-b7c691fafb26" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1813.323969] env[67752]: DEBUG nova.network.neutron [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Updating instance_info_cache with network_info: [{"id": "1dbe4598-b834-41f2-a435-4cadf5bb8bda", "address": "fa:16:3e:52:43:2e", "network": {"id": "b684aefa-3309-4e2f-a55b-32853fdd1a0d", "bridge": "br-int", "label": "tempest-AttachVolumeTestJSON-1831973259-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "dc0e175514604a85bb7d9284ff4be671", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "257e5ea7-8b80-4301-9900-a754f1fe2031", "external-id": "nsx-vlan-transportzone-682", "segmentation_id": 682, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap1dbe4598-b8", "ovs_interfaceid": "1dbe4598-b834-41f2-a435-4cadf5bb8bda", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1813.339222] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Releasing lock "refresh_cache-f2509f7b-1f6b-4942-b591-b7c691fafb26" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1813.339535] env[67752]: DEBUG nova.compute.manager [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Instance network_info: |[{"id": "1dbe4598-b834-41f2-a435-4cadf5bb8bda", "address": "fa:16:3e:52:43:2e", "network": {"id": "b684aefa-3309-4e2f-a55b-32853fdd1a0d", "bridge": "br-int", "label": "tempest-AttachVolumeTestJSON-1831973259-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "dc0e175514604a85bb7d9284ff4be671", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "257e5ea7-8b80-4301-9900-a754f1fe2031", "external-id": "nsx-vlan-transportzone-682", "segmentation_id": 682, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap1dbe4598-b8", "ovs_interfaceid": "1dbe4598-b834-41f2-a435-4cadf5bb8bda", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1813.339839] env[67752]: DEBUG oslo_concurrency.lockutils [req-54ede0e6-c120-4fc8-880a-64ebbcfc6cec req-82cdca3c-a6e2-482c-84c6-579eed1822bf service nova] Acquired lock "refresh_cache-f2509f7b-1f6b-4942-b591-b7c691fafb26" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1813.340030] env[67752]: DEBUG nova.network.neutron [req-54ede0e6-c120-4fc8-880a-64ebbcfc6cec req-82cdca3c-a6e2-482c-84c6-579eed1822bf service nova] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Refreshing network info cache for port 1dbe4598-b834-41f2-a435-4cadf5bb8bda {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1813.341115] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:52:43:2e', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '257e5ea7-8b80-4301-9900-a754f1fe2031', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '1dbe4598-b834-41f2-a435-4cadf5bb8bda', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1813.350052] env[67752]: DEBUG oslo.service.loopingcall [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1813.352914] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1813.353378] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-bc5f20ee-82c2-4d31-b105-03e5a376b4dc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1813.374376] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1813.374376] env[67752]: value = "task-3199821" [ 1813.374376] env[67752]: _type = "Task" [ 1813.374376] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1813.381673] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199821, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1813.654912] env[67752]: DEBUG nova.network.neutron [req-54ede0e6-c120-4fc8-880a-64ebbcfc6cec req-82cdca3c-a6e2-482c-84c6-579eed1822bf service nova] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Updated VIF entry in instance network info cache for port 1dbe4598-b834-41f2-a435-4cadf5bb8bda. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1813.655341] env[67752]: DEBUG nova.network.neutron [req-54ede0e6-c120-4fc8-880a-64ebbcfc6cec req-82cdca3c-a6e2-482c-84c6-579eed1822bf service nova] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Updating instance_info_cache with network_info: [{"id": "1dbe4598-b834-41f2-a435-4cadf5bb8bda", "address": "fa:16:3e:52:43:2e", "network": {"id": "b684aefa-3309-4e2f-a55b-32853fdd1a0d", "bridge": "br-int", "label": "tempest-AttachVolumeTestJSON-1831973259-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.9", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "dc0e175514604a85bb7d9284ff4be671", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "257e5ea7-8b80-4301-9900-a754f1fe2031", "external-id": "nsx-vlan-transportzone-682", "segmentation_id": 682, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap1dbe4598-b8", "ovs_interfaceid": "1dbe4598-b834-41f2-a435-4cadf5bb8bda", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1813.664649] env[67752]: DEBUG oslo_concurrency.lockutils [req-54ede0e6-c120-4fc8-880a-64ebbcfc6cec req-82cdca3c-a6e2-482c-84c6-579eed1822bf service nova] Releasing lock "refresh_cache-f2509f7b-1f6b-4942-b591-b7c691fafb26" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1813.883506] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199821, 'name': CreateVM_Task, 'duration_secs': 0.272979} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1813.883678] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1813.884378] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1813.884550] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1813.884887] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1813.885146] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-8e78ab8a-a5b3-492e-bdd8-0225f064a11d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1813.890034] env[67752]: DEBUG oslo_vmware.api [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Waiting for the task: (returnval){ [ 1813.890034] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]529aebdc-6783-5eae-655a-9533cab0196d" [ 1813.890034] env[67752]: _type = "Task" [ 1813.890034] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1813.896729] env[67752]: DEBUG oslo_vmware.api [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]529aebdc-6783-5eae-655a-9533cab0196d, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1814.399595] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1814.399595] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1814.399595] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1815.634743] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1816.872350] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7fd00b13-f662-4a6b-a0c6-b0feca8719ba tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquiring lock "d52496eb-f8e3-4d86-8454-f56e9097777e" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1817.635341] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1817.646712] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1817.646918] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1817.647099] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1817.647269] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1817.648353] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c48ec216-b38d-4276-9e09-1653a2906cb2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1817.656863] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5ddd136c-494b-474a-9ac6-0d6d6334b87e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1817.670106] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cf107573-02f9-4348-be7a-db42f293f248 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1817.675976] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa7aeac4-4e3c-4ff9-8be7-0209104a48f1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1817.705631] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180966MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1817.705777] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1817.705995] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1817.775888] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 81738fe6-6d50-4dfe-ae86-4c8771837168 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1817.776062] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e4ebb0f0-ab58-491f-bc4a-7548bbe415d8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1817.776214] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c1975c17-15a5-4751-bf7f-d5b7e0d2fea1 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1817.776316] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1817.776434] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 7557e3a1-b298-401f-8320-5a48ddda3298 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1817.776554] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 34c75d7e-e63e-4eff-afad-0edd61e023fc actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1817.776668] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d52496eb-f8e3-4d86-8454-f56e9097777e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1817.776782] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 09e003b7-3c4c-4b11-a52d-749acf709068 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1817.776897] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fea66a16-92c3-4c7a-8bff-c6eb59270158 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1817.777033] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f2509f7b-1f6b-4942-b591-b7c691fafb26 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1817.777270] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1817.777417] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1817.898905] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b6d86155-7ce8-41f2-8421-d5dc7928ae8e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1817.907085] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e655429e-d8cf-4b81-b19f-005d8c93ce2f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1817.935920] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a114eca5-5d63-466e-acea-183a1891ce69 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1817.942841] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-64bb6cea-517f-4414-b6a2-53918f74c53f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1817.956423] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1817.967371] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1817.982277] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1817.982468] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.276s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1834.184403] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "369dbb1f-3de0-4ba9-af2b-98167a6b4508" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1834.184738] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "369dbb1f-3de0-4ba9-af2b-98167a6b4508" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1844.566881] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f1dea674-e4b0-44ba-87de-044ddc21a1c4 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "09e003b7-3c4c-4b11-a52d-749acf709068" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1853.413616] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f5471ecc-2e83-4753-89bf-69c063096ea9 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Acquiring lock "fea66a16-92c3-4c7a-8bff-c6eb59270158" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1858.891548] env[67752]: WARNING oslo_vmware.rw_handles [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1858.891548] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1858.891548] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1858.891548] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1858.891548] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1858.891548] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1858.891548] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1858.891548] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1858.891548] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1858.891548] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1858.891548] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1858.891548] env[67752]: ERROR oslo_vmware.rw_handles [ 1858.892402] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/69a675df-9c44-40fe-8d8a-7bb85a7a067e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1858.894181] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1858.894461] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Copying Virtual Disk [datastore2] vmware_temp/69a675df-9c44-40fe-8d8a-7bb85a7a067e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/69a675df-9c44-40fe-8d8a-7bb85a7a067e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1858.894751] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-f28736b4-4b41-48a8-9fef-72b9f3947f56 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1858.901942] env[67752]: DEBUG oslo_vmware.api [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Waiting for the task: (returnval){ [ 1858.901942] env[67752]: value = "task-3199822" [ 1858.901942] env[67752]: _type = "Task" [ 1858.901942] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1858.909968] env[67752]: DEBUG oslo_vmware.api [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Task: {'id': task-3199822, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1859.412573] env[67752]: DEBUG oslo_vmware.exceptions [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1859.412875] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1859.413464] env[67752]: ERROR nova.compute.manager [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1859.413464] env[67752]: Faults: ['InvalidArgument'] [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Traceback (most recent call last): [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] yield resources [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] self.driver.spawn(context, instance, image_meta, [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] self._fetch_image_if_missing(context, vi) [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] image_cache(vi, tmp_image_ds_loc) [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] vm_util.copy_virtual_disk( [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] session._wait_for_task(vmdk_copy_task) [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] return self.wait_for_task(task_ref) [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] return evt.wait() [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] result = hub.switch() [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] return self.greenlet.switch() [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] self.f(*self.args, **self.kw) [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] raise exceptions.translate_fault(task_info.error) [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Faults: ['InvalidArgument'] [ 1859.413464] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] [ 1859.414617] env[67752]: INFO nova.compute.manager [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Terminating instance [ 1859.415373] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1859.415595] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1859.415839] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-5f2eb5e0-f356-4a4a-8fc7-d8d7ce66162b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1859.419359] env[67752]: DEBUG nova.compute.manager [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1859.419551] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1859.420293] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-96d6d95c-ddd0-41e3-b6fe-2acb796e919b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1859.427257] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1859.427519] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-f359d3b5-ca31-44f9-8e43-2fd72fb0b752 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1859.429823] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1859.429995] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1859.430995] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-c89590bf-8d25-4274-a9a1-fffa25568403 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1859.435855] env[67752]: DEBUG oslo_vmware.api [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Waiting for the task: (returnval){ [ 1859.435855] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52542d19-1604-975f-2b60-ef1b7fff450d" [ 1859.435855] env[67752]: _type = "Task" [ 1859.435855] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1859.443089] env[67752]: DEBUG oslo_vmware.api [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52542d19-1604-975f-2b60-ef1b7fff450d, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1859.566917] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1859.567111] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1859.567288] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Deleting the datastore file [datastore2] 81738fe6-6d50-4dfe-ae86-4c8771837168 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1859.567558] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-9f65dda4-cbce-439f-9501-562923a4ba06 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1859.574319] env[67752]: DEBUG oslo_vmware.api [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Waiting for the task: (returnval){ [ 1859.574319] env[67752]: value = "task-3199824" [ 1859.574319] env[67752]: _type = "Task" [ 1859.574319] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1859.582718] env[67752]: DEBUG oslo_vmware.api [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Task: {'id': task-3199824, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1859.947239] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1859.947239] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Creating directory with path [datastore2] vmware_temp/e3383b74-330a-4f6f-9af1-cad2acebd2fd/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1859.947530] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-fb14a903-39fc-4ca5-9fda-2e21ba099ae9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1859.959471] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Created directory with path [datastore2] vmware_temp/e3383b74-330a-4f6f-9af1-cad2acebd2fd/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1859.959695] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Fetch image to [datastore2] vmware_temp/e3383b74-330a-4f6f-9af1-cad2acebd2fd/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1859.959874] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/e3383b74-330a-4f6f-9af1-cad2acebd2fd/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1859.960682] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-27cd8484-b194-4704-bc44-71e21ae61ac4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1859.967891] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-71fa5c13-4196-4ec6-bd45-6a7369e9e434 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1859.976980] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6478c6ef-e013-4f47-9eb0-06cdf183f4ec {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1860.007076] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6f36d7f6-b040-4fb7-8507-ff01aa5b0f28 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1860.013143] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-d167e767-cd35-4b0a-a859-9554d817a11a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1860.038805] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1860.083520] env[67752]: DEBUG oslo_vmware.api [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Task: {'id': task-3199824, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.086882} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1860.084211] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1860.084211] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1860.084211] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1860.084430] env[67752]: INFO nova.compute.manager [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Took 0.66 seconds to destroy the instance on the hypervisor. [ 1860.086796] env[67752]: DEBUG nova.compute.claims [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1860.086971] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1860.087243] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1860.170723] env[67752]: DEBUG oslo_vmware.rw_handles [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/e3383b74-330a-4f6f-9af1-cad2acebd2fd/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1860.232028] env[67752]: DEBUG oslo_vmware.rw_handles [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1860.232228] env[67752]: DEBUG oslo_vmware.rw_handles [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Closing write handle for https://esx7c2n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/e3383b74-330a-4f6f-9af1-cad2acebd2fd/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1860.321013] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-50beb5fb-ee92-4d63-a305-8551004ce557 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1860.328478] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bd63f119-bc9a-4abb-a4ce-d91c3f7b10a8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1860.357371] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-98e7e5c2-bf36-49d8-be5d-030ee9af8d28 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1860.363774] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7436ca4a-89c3-41e7-bb6b-cbc995a5921a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1860.376084] env[67752]: DEBUG nova.compute.provider_tree [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1860.384888] env[67752]: DEBUG nova.scheduler.client.report [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1860.398882] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.312s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1860.399473] env[67752]: ERROR nova.compute.manager [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1860.399473] env[67752]: Faults: ['InvalidArgument'] [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Traceback (most recent call last): [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] self.driver.spawn(context, instance, image_meta, [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] self._fetch_image_if_missing(context, vi) [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] image_cache(vi, tmp_image_ds_loc) [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] vm_util.copy_virtual_disk( [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] session._wait_for_task(vmdk_copy_task) [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] return self.wait_for_task(task_ref) [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] return evt.wait() [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] result = hub.switch() [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] return self.greenlet.switch() [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] self.f(*self.args, **self.kw) [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] raise exceptions.translate_fault(task_info.error) [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Faults: ['InvalidArgument'] [ 1860.399473] env[67752]: ERROR nova.compute.manager [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] [ 1860.400482] env[67752]: DEBUG nova.compute.utils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1860.401802] env[67752]: DEBUG nova.compute.manager [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Build of instance 81738fe6-6d50-4dfe-ae86-4c8771837168 was re-scheduled: A specified parameter was not correct: fileType [ 1860.401802] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1860.402191] env[67752]: DEBUG nova.compute.manager [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1860.402367] env[67752]: DEBUG nova.compute.manager [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1860.402553] env[67752]: DEBUG nova.compute.manager [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1860.402734] env[67752]: DEBUG nova.network.neutron [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1861.412371] env[67752]: DEBUG nova.network.neutron [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1861.423176] env[67752]: INFO nova.compute.manager [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Took 1.02 seconds to deallocate network for instance. [ 1861.518852] env[67752]: INFO nova.scheduler.client.report [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Deleted allocations for instance 81738fe6-6d50-4dfe-ae86-4c8771837168 [ 1861.539317] env[67752]: DEBUG oslo_concurrency.lockutils [None req-33312cda-0622-4c0f-8ae1-f9c221292917 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "81738fe6-6d50-4dfe-ae86-4c8771837168" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 635.198s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1861.541035] env[67752]: DEBUG oslo_concurrency.lockutils [None req-70799d8f-a860-4c0e-b81a-d37df565547d tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "81738fe6-6d50-4dfe-ae86-4c8771837168" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 439.788s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1861.541035] env[67752]: DEBUG oslo_concurrency.lockutils [None req-70799d8f-a860-4c0e-b81a-d37df565547d tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquiring lock "81738fe6-6d50-4dfe-ae86-4c8771837168-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1861.541035] env[67752]: DEBUG oslo_concurrency.lockutils [None req-70799d8f-a860-4c0e-b81a-d37df565547d tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "81738fe6-6d50-4dfe-ae86-4c8771837168-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1861.541035] env[67752]: DEBUG oslo_concurrency.lockutils [None req-70799d8f-a860-4c0e-b81a-d37df565547d tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "81738fe6-6d50-4dfe-ae86-4c8771837168-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1861.542974] env[67752]: INFO nova.compute.manager [None req-70799d8f-a860-4c0e-b81a-d37df565547d tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Terminating instance [ 1861.544687] env[67752]: DEBUG nova.compute.manager [None req-70799d8f-a860-4c0e-b81a-d37df565547d tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1861.544878] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-70799d8f-a860-4c0e-b81a-d37df565547d tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1861.545360] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-047de687-0025-409f-86d2-2509f885ca62 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1861.552177] env[67752]: DEBUG nova.compute.manager [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1861.557548] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ec86cbdf-5605-4daf-b8f5-5dc757afb7de {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1861.585741] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-70799d8f-a860-4c0e-b81a-d37df565547d tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 81738fe6-6d50-4dfe-ae86-4c8771837168 could not be found. [ 1861.585950] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-70799d8f-a860-4c0e-b81a-d37df565547d tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1861.586145] env[67752]: INFO nova.compute.manager [None req-70799d8f-a860-4c0e-b81a-d37df565547d tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1861.586389] env[67752]: DEBUG oslo.service.loopingcall [None req-70799d8f-a860-4c0e-b81a-d37df565547d tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1861.588629] env[67752]: DEBUG nova.compute.manager [-] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1861.588733] env[67752]: DEBUG nova.network.neutron [-] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1861.601902] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1861.602147] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1861.603658] env[67752]: INFO nova.compute.claims [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1861.613812] env[67752]: DEBUG nova.network.neutron [-] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1861.623404] env[67752]: INFO nova.compute.manager [-] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] Took 0.03 seconds to deallocate network for instance. [ 1861.712261] env[67752]: DEBUG oslo_concurrency.lockutils [None req-70799d8f-a860-4c0e-b81a-d37df565547d tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "81738fe6-6d50-4dfe-ae86-4c8771837168" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.172s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1861.713242] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "81738fe6-6d50-4dfe-ae86-4c8771837168" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 69.805s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1861.713472] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 81738fe6-6d50-4dfe-ae86-4c8771837168] During sync_power_state the instance has a pending task (deleting). Skip. [ 1861.713618] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "81738fe6-6d50-4dfe-ae86-4c8771837168" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1861.774083] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-725f8411-d793-4170-a32d-a9bd11b2b6ba {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1861.781558] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79b68d86-b2c6-40bb-9a73-e46bb6831876 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1861.811738] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-99531dac-a0e5-4373-be20-bbc6699e5233 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1861.818380] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7e2be683-44cf-4177-9324-27cdfd05ab1d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1861.831108] env[67752]: DEBUG nova.compute.provider_tree [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1861.840754] env[67752]: DEBUG nova.scheduler.client.report [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1861.855378] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.253s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1861.855854] env[67752]: DEBUG nova.compute.manager [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1861.888166] env[67752]: DEBUG nova.compute.utils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1861.889624] env[67752]: DEBUG nova.compute.manager [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1861.889906] env[67752]: DEBUG nova.network.neutron [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1861.900073] env[67752]: DEBUG nova.compute.manager [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1861.954786] env[67752]: DEBUG nova.policy [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '641d021b5cfb4a3885a3db7932407a38', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'e117ebfe10cb4714b149d6e9bc132273', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1861.962318] env[67752]: DEBUG nova.compute.manager [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1861.988995] env[67752]: DEBUG nova.virt.hardware [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1861.989262] env[67752]: DEBUG nova.virt.hardware [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1861.989602] env[67752]: DEBUG nova.virt.hardware [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1861.989824] env[67752]: DEBUG nova.virt.hardware [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1861.989985] env[67752]: DEBUG nova.virt.hardware [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1861.990161] env[67752]: DEBUG nova.virt.hardware [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1861.990512] env[67752]: DEBUG nova.virt.hardware [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1861.990685] env[67752]: DEBUG nova.virt.hardware [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1861.990857] env[67752]: DEBUG nova.virt.hardware [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1861.991035] env[67752]: DEBUG nova.virt.hardware [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1861.991215] env[67752]: DEBUG nova.virt.hardware [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1861.992152] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-415ede50-832d-4349-b6b9-b446a942717d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1862.002799] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9bc63c53-2330-428a-94e9-ec1c6788211b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1862.443503] env[67752]: DEBUG nova.network.neutron [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Successfully created port: b8e159c1-5e87-4684-b9ca-642824a429fe {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 1863.117280] env[67752]: DEBUG nova.network.neutron [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Successfully updated port: b8e159c1-5e87-4684-b9ca-642824a429fe {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 1863.129504] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "refresh_cache-369dbb1f-3de0-4ba9-af2b-98167a6b4508" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1863.129679] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquired lock "refresh_cache-369dbb1f-3de0-4ba9-af2b-98167a6b4508" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1863.129834] env[67752]: DEBUG nova.network.neutron [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 1863.173481] env[67752]: DEBUG nova.network.neutron [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1863.465358] env[67752]: DEBUG nova.network.neutron [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Updating instance_info_cache with network_info: [{"id": "b8e159c1-5e87-4684-b9ca-642824a429fe", "address": "fa:16:3e:b2:a5:75", "network": {"id": "48a676ef-ba7b-476c-8ee5-40eb99a49ec1", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-1953737432-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e117ebfe10cb4714b149d6e9bc132273", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "b36c5ae6-c344-4bd1-8239-29128e2bbfbf", "external-id": "nsx-vlan-transportzone-214", "segmentation_id": 214, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapb8e159c1-5e", "ovs_interfaceid": "b8e159c1-5e87-4684-b9ca-642824a429fe", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1863.478634] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Releasing lock "refresh_cache-369dbb1f-3de0-4ba9-af2b-98167a6b4508" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1863.479012] env[67752]: DEBUG nova.compute.manager [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Instance network_info: |[{"id": "b8e159c1-5e87-4684-b9ca-642824a429fe", "address": "fa:16:3e:b2:a5:75", "network": {"id": "48a676ef-ba7b-476c-8ee5-40eb99a49ec1", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-1953737432-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e117ebfe10cb4714b149d6e9bc132273", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "b36c5ae6-c344-4bd1-8239-29128e2bbfbf", "external-id": "nsx-vlan-transportzone-214", "segmentation_id": 214, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapb8e159c1-5e", "ovs_interfaceid": "b8e159c1-5e87-4684-b9ca-642824a429fe", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 1863.479499] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:b2:a5:75', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'b36c5ae6-c344-4bd1-8239-29128e2bbfbf', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'b8e159c1-5e87-4684-b9ca-642824a429fe', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 1863.487659] env[67752]: DEBUG oslo.service.loopingcall [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1863.488257] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 1863.488458] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-f0dd86c8-0a1c-4474-9d8b-4827fc504c13 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1863.511513] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 1863.511513] env[67752]: value = "task-3199825" [ 1863.511513] env[67752]: _type = "Task" [ 1863.511513] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1863.520709] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199825, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1864.022427] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199825, 'name': CreateVM_Task, 'duration_secs': 0.414554} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1864.022595] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 1864.035868] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1864.036080] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1864.036374] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 1864.036610] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-a124544b-440f-4ea7-aac7-17ea07177e57 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1864.041303] env[67752]: DEBUG oslo_vmware.api [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Waiting for the task: (returnval){ [ 1864.041303] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52f07d0a-4cda-ad04-2a79-10f2300716ce" [ 1864.041303] env[67752]: _type = "Task" [ 1864.041303] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1864.050406] env[67752]: DEBUG oslo_vmware.api [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52f07d0a-4cda-ad04-2a79-10f2300716ce, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1864.550892] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1864.551239] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 1864.551353] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1864.817996] env[67752]: DEBUG nova.compute.manager [req-cc580790-aa6c-4a71-9bb2-9921c1ac20b2 req-f181b815-a5d3-4e98-989f-f9ace45fc037 service nova] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Received event network-vif-plugged-b8e159c1-5e87-4684-b9ca-642824a429fe {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1864.818310] env[67752]: DEBUG oslo_concurrency.lockutils [req-cc580790-aa6c-4a71-9bb2-9921c1ac20b2 req-f181b815-a5d3-4e98-989f-f9ace45fc037 service nova] Acquiring lock "369dbb1f-3de0-4ba9-af2b-98167a6b4508-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1864.818552] env[67752]: DEBUG oslo_concurrency.lockutils [req-cc580790-aa6c-4a71-9bb2-9921c1ac20b2 req-f181b815-a5d3-4e98-989f-f9ace45fc037 service nova] Lock "369dbb1f-3de0-4ba9-af2b-98167a6b4508-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1864.818734] env[67752]: DEBUG oslo_concurrency.lockutils [req-cc580790-aa6c-4a71-9bb2-9921c1ac20b2 req-f181b815-a5d3-4e98-989f-f9ace45fc037 service nova] Lock "369dbb1f-3de0-4ba9-af2b-98167a6b4508-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1864.818907] env[67752]: DEBUG nova.compute.manager [req-cc580790-aa6c-4a71-9bb2-9921c1ac20b2 req-f181b815-a5d3-4e98-989f-f9ace45fc037 service nova] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] No waiting events found dispatching network-vif-plugged-b8e159c1-5e87-4684-b9ca-642824a429fe {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 1864.819085] env[67752]: WARNING nova.compute.manager [req-cc580790-aa6c-4a71-9bb2-9921c1ac20b2 req-f181b815-a5d3-4e98-989f-f9ace45fc037 service nova] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Received unexpected event network-vif-plugged-b8e159c1-5e87-4684-b9ca-642824a429fe for instance with vm_state building and task_state spawning. [ 1864.819276] env[67752]: DEBUG nova.compute.manager [req-cc580790-aa6c-4a71-9bb2-9921c1ac20b2 req-f181b815-a5d3-4e98-989f-f9ace45fc037 service nova] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Received event network-changed-b8e159c1-5e87-4684-b9ca-642824a429fe {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 1864.819472] env[67752]: DEBUG nova.compute.manager [req-cc580790-aa6c-4a71-9bb2-9921c1ac20b2 req-f181b815-a5d3-4e98-989f-f9ace45fc037 service nova] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Refreshing instance network info cache due to event network-changed-b8e159c1-5e87-4684-b9ca-642824a429fe. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 1864.819714] env[67752]: DEBUG oslo_concurrency.lockutils [req-cc580790-aa6c-4a71-9bb2-9921c1ac20b2 req-f181b815-a5d3-4e98-989f-f9ace45fc037 service nova] Acquiring lock "refresh_cache-369dbb1f-3de0-4ba9-af2b-98167a6b4508" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 1864.819865] env[67752]: DEBUG oslo_concurrency.lockutils [req-cc580790-aa6c-4a71-9bb2-9921c1ac20b2 req-f181b815-a5d3-4e98-989f-f9ace45fc037 service nova] Acquired lock "refresh_cache-369dbb1f-3de0-4ba9-af2b-98167a6b4508" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1864.820035] env[67752]: DEBUG nova.network.neutron [req-cc580790-aa6c-4a71-9bb2-9921c1ac20b2 req-f181b815-a5d3-4e98-989f-f9ace45fc037 service nova] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Refreshing network info cache for port b8e159c1-5e87-4684-b9ca-642824a429fe {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 1864.977696] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1865.110956] env[67752]: DEBUG nova.network.neutron [req-cc580790-aa6c-4a71-9bb2-9921c1ac20b2 req-f181b815-a5d3-4e98-989f-f9ace45fc037 service nova] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Updated VIF entry in instance network info cache for port b8e159c1-5e87-4684-b9ca-642824a429fe. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 1865.111339] env[67752]: DEBUG nova.network.neutron [req-cc580790-aa6c-4a71-9bb2-9921c1ac20b2 req-f181b815-a5d3-4e98-989f-f9ace45fc037 service nova] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Updating instance_info_cache with network_info: [{"id": "b8e159c1-5e87-4684-b9ca-642824a429fe", "address": "fa:16:3e:b2:a5:75", "network": {"id": "48a676ef-ba7b-476c-8ee5-40eb99a49ec1", "bridge": "br-int", "label": "tempest-AttachVolumeNegativeTest-1953737432-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.3", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "e117ebfe10cb4714b149d6e9bc132273", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "b36c5ae6-c344-4bd1-8239-29128e2bbfbf", "external-id": "nsx-vlan-transportzone-214", "segmentation_id": 214, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapb8e159c1-5e", "ovs_interfaceid": "b8e159c1-5e87-4684-b9ca-642824a429fe", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1865.120533] env[67752]: DEBUG oslo_concurrency.lockutils [req-cc580790-aa6c-4a71-9bb2-9921c1ac20b2 req-f181b815-a5d3-4e98-989f-f9ace45fc037 service nova] Releasing lock "refresh_cache-369dbb1f-3de0-4ba9-af2b-98167a6b4508" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1870.635820] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1870.635820] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1870.635820] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1870.635820] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 1871.635260] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1871.635511] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1873.631356] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1873.656564] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1873.656740] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 1873.656848] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 1873.678924] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1873.678924] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1873.678924] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1873.678924] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1873.678924] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1873.678924] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1873.679521] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1873.680555] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1873.680555] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1873.680555] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1873.680555] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 1875.634991] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1879.634406] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1879.647531] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1879.647531] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1879.647531] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1879.647531] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1879.648489] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b44c66de-28d5-47d4-a5b1-25bf0bbce9bc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.657014] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f69a926a-06b5-4927-8671-3fd987957c44 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.671340] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a15e72e9-8e3e-4ae8-88b6-8c597bdf2225 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.677304] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e05e224e-bbda-4069-9ee0-c68f072c7193 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.707135] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181011MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1879.707301] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1879.707500] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1879.784131] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e4ebb0f0-ab58-491f-bc4a-7548bbe415d8 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1879.784289] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c1975c17-15a5-4751-bf7f-d5b7e0d2fea1 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1879.784289] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1879.784446] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 7557e3a1-b298-401f-8320-5a48ddda3298 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1879.784501] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 34c75d7e-e63e-4eff-afad-0edd61e023fc actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1879.784616] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d52496eb-f8e3-4d86-8454-f56e9097777e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1879.784738] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 09e003b7-3c4c-4b11-a52d-749acf709068 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1879.784856] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fea66a16-92c3-4c7a-8bff-c6eb59270158 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1879.784972] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f2509f7b-1f6b-4942-b591-b7c691fafb26 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1879.785102] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 369dbb1f-3de0-4ba9-af2b-98167a6b4508 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1879.785304] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1879.785441] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1879.905384] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-82a978a2-81a1-41b9-87cb-6b07c3467007 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.912816] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a7e2eead-b1e4-412c-b5db-70cb7904f3ff {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.941306] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b58ac641-207d-42b4-9532-210cbbdd61bc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.947711] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-91cdeea4-b8e1-4a54-ad79-85de6b1a6dd3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1879.962378] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1879.970542] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1879.984898] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1879.985100] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.278s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1907.375911] env[67752]: WARNING oslo_vmware.rw_handles [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1907.375911] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1907.375911] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1907.375911] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1907.375911] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1907.375911] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1907.375911] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1907.375911] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1907.375911] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1907.375911] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1907.375911] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1907.375911] env[67752]: ERROR oslo_vmware.rw_handles [ 1907.376671] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/e3383b74-330a-4f6f-9af1-cad2acebd2fd/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1907.379170] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1907.379443] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Copying Virtual Disk [datastore2] vmware_temp/e3383b74-330a-4f6f-9af1-cad2acebd2fd/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/e3383b74-330a-4f6f-9af1-cad2acebd2fd/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1907.379783] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-c61b40d0-73ae-4eae-8936-c6265830e058 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1907.388301] env[67752]: DEBUG oslo_vmware.api [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Waiting for the task: (returnval){ [ 1907.388301] env[67752]: value = "task-3199826" [ 1907.388301] env[67752]: _type = "Task" [ 1907.388301] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1907.396093] env[67752]: DEBUG oslo_vmware.api [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Task: {'id': task-3199826, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1907.899345] env[67752]: DEBUG oslo_vmware.exceptions [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1907.899609] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1907.900202] env[67752]: ERROR nova.compute.manager [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1907.900202] env[67752]: Faults: ['InvalidArgument'] [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Traceback (most recent call last): [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] yield resources [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] self.driver.spawn(context, instance, image_meta, [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] self._fetch_image_if_missing(context, vi) [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] image_cache(vi, tmp_image_ds_loc) [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] vm_util.copy_virtual_disk( [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] session._wait_for_task(vmdk_copy_task) [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] return self.wait_for_task(task_ref) [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] return evt.wait() [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] result = hub.switch() [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] return self.greenlet.switch() [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] self.f(*self.args, **self.kw) [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] raise exceptions.translate_fault(task_info.error) [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Faults: ['InvalidArgument'] [ 1907.900202] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] [ 1907.901410] env[67752]: INFO nova.compute.manager [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Terminating instance [ 1907.902155] env[67752]: DEBUG oslo_concurrency.lockutils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1907.902414] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1907.902680] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-93d3ad2d-2e08-4eff-be42-1c687b47052e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1907.904832] env[67752]: DEBUG nova.compute.manager [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1907.905053] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1907.905845] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5045a0b9-f6b3-4b53-a9e3-f630ef72f7e9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1907.912486] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1907.912740] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-01680bd5-df98-497c-8d30-9734943029e4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1907.914745] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1907.914931] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1907.915819] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-b43883cd-f770-4cf2-a27a-f2a2397bc088 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1907.920173] env[67752]: DEBUG oslo_vmware.api [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Waiting for the task: (returnval){ [ 1907.920173] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52d1f0fd-0680-81b5-1c6b-cd87e19021fc" [ 1907.920173] env[67752]: _type = "Task" [ 1907.920173] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1907.927134] env[67752]: DEBUG oslo_vmware.api [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52d1f0fd-0680-81b5-1c6b-cd87e19021fc, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1907.983091] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1907.983297] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1907.983499] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Deleting the datastore file [datastore2] e4ebb0f0-ab58-491f-bc4a-7548bbe415d8 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1907.983783] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-3c5d69ba-c4d4-49eb-9e8b-a557211bacf2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1907.991043] env[67752]: DEBUG oslo_vmware.api [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Waiting for the task: (returnval){ [ 1907.991043] env[67752]: value = "task-3199828" [ 1907.991043] env[67752]: _type = "Task" [ 1907.991043] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1907.998594] env[67752]: DEBUG oslo_vmware.api [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Task: {'id': task-3199828, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1908.431160] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1908.431438] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Creating directory with path [datastore2] vmware_temp/9189705b-b551-4dc0-afd6-86cbcf65bb3b/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1908.431669] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-8663d7a9-6343-4b23-bb60-2cad0b8ce2bb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1908.442801] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Created directory with path [datastore2] vmware_temp/9189705b-b551-4dc0-afd6-86cbcf65bb3b/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1908.443008] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Fetch image to [datastore2] vmware_temp/9189705b-b551-4dc0-afd6-86cbcf65bb3b/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1908.443192] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/9189705b-b551-4dc0-afd6-86cbcf65bb3b/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1908.443978] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c89d22e8-0e1b-4711-8214-5f1d7bc114fc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1908.450599] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf1f1bf1-c6c2-4fa9-bbc3-60238f371da0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1908.459374] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f0411130-3fa6-44b0-99d4-7e2d2d29b823 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1908.490505] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fee7ed99-ef1c-4d2f-8fd7-23720c77a437 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1908.501076] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-aa336ddb-e801-4652-880d-732911499b90 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1908.502745] env[67752]: DEBUG oslo_vmware.api [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Task: {'id': task-3199828, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.06627} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1908.502992] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1908.503195] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1908.503372] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1908.503547] env[67752]: INFO nova.compute.manager [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1908.505819] env[67752]: DEBUG nova.compute.claims [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1908.506028] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1908.506256] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1908.527064] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1908.607395] env[67752]: DEBUG oslo_vmware.rw_handles [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/9189705b-b551-4dc0-afd6-86cbcf65bb3b/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1908.668919] env[67752]: DEBUG oslo_vmware.rw_handles [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1908.669131] env[67752]: DEBUG oslo_vmware.rw_handles [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Closing write handle for https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/9189705b-b551-4dc0-afd6-86cbcf65bb3b/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1908.727891] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-abb7e86e-8f44-4b3f-92bb-a627fb1ef496 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1908.735703] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b31d6bb1-a6a3-4673-b724-da27e8d58d1d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1908.765233] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-faa89bff-6a6e-4b0c-b921-d55f8298ef01 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1908.771705] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb02208d-4b77-4b68-91f4-669744a53f04 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1908.784233] env[67752]: DEBUG nova.compute.provider_tree [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1908.792262] env[67752]: DEBUG nova.scheduler.client.report [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1908.805805] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.299s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1908.806383] env[67752]: ERROR nova.compute.manager [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1908.806383] env[67752]: Faults: ['InvalidArgument'] [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Traceback (most recent call last): [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] self.driver.spawn(context, instance, image_meta, [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] self._fetch_image_if_missing(context, vi) [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] image_cache(vi, tmp_image_ds_loc) [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] vm_util.copy_virtual_disk( [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] session._wait_for_task(vmdk_copy_task) [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] return self.wait_for_task(task_ref) [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] return evt.wait() [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] result = hub.switch() [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] return self.greenlet.switch() [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] self.f(*self.args, **self.kw) [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] raise exceptions.translate_fault(task_info.error) [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Faults: ['InvalidArgument'] [ 1908.806383] env[67752]: ERROR nova.compute.manager [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] [ 1908.807396] env[67752]: DEBUG nova.compute.utils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1908.808414] env[67752]: DEBUG nova.compute.manager [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Build of instance e4ebb0f0-ab58-491f-bc4a-7548bbe415d8 was re-scheduled: A specified parameter was not correct: fileType [ 1908.808414] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1908.808772] env[67752]: DEBUG nova.compute.manager [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1908.808949] env[67752]: DEBUG nova.compute.manager [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1908.809138] env[67752]: DEBUG nova.compute.manager [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1908.809303] env[67752]: DEBUG nova.network.neutron [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1909.113032] env[67752]: DEBUG nova.network.neutron [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1909.123245] env[67752]: INFO nova.compute.manager [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Took 0.31 seconds to deallocate network for instance. [ 1909.217906] env[67752]: INFO nova.scheduler.client.report [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Deleted allocations for instance e4ebb0f0-ab58-491f-bc4a-7548bbe415d8 [ 1909.241186] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bf0c97f4-6122-46be-88dd-e2dfc8c2b87f tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Lock "e4ebb0f0-ab58-491f-bc4a-7548bbe415d8" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 669.317s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1909.241397] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3f54ff6d-85e1-4d00-be13-bfc4a9353992 tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Lock "e4ebb0f0-ab58-491f-bc4a-7548bbe415d8" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 473.932s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1909.241633] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3f54ff6d-85e1-4d00-be13-bfc4a9353992 tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Acquiring lock "e4ebb0f0-ab58-491f-bc4a-7548bbe415d8-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1909.241846] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3f54ff6d-85e1-4d00-be13-bfc4a9353992 tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Lock "e4ebb0f0-ab58-491f-bc4a-7548bbe415d8-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1909.242037] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3f54ff6d-85e1-4d00-be13-bfc4a9353992 tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Lock "e4ebb0f0-ab58-491f-bc4a-7548bbe415d8-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1909.245037] env[67752]: INFO nova.compute.manager [None req-3f54ff6d-85e1-4d00-be13-bfc4a9353992 tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Terminating instance [ 1909.246061] env[67752]: DEBUG nova.compute.manager [None req-3f54ff6d-85e1-4d00-be13-bfc4a9353992 tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1909.246279] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-3f54ff6d-85e1-4d00-be13-bfc4a9353992 tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1909.246756] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-5cf18db9-3fe1-49d3-a824-e27c5ec8a427 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1909.256137] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2596721f-c3cf-46ab-868d-ebd729c5a7bd {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1909.285282] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-3f54ff6d-85e1-4d00-be13-bfc4a9353992 tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance e4ebb0f0-ab58-491f-bc4a-7548bbe415d8 could not be found. [ 1909.285525] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-3f54ff6d-85e1-4d00-be13-bfc4a9353992 tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1909.285709] env[67752]: INFO nova.compute.manager [None req-3f54ff6d-85e1-4d00-be13-bfc4a9353992 tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1909.285959] env[67752]: DEBUG oslo.service.loopingcall [None req-3f54ff6d-85e1-4d00-be13-bfc4a9353992 tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1909.286214] env[67752]: DEBUG nova.compute.manager [-] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1909.286300] env[67752]: DEBUG nova.network.neutron [-] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1909.338025] env[67752]: DEBUG nova.network.neutron [-] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1909.346145] env[67752]: INFO nova.compute.manager [-] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] Took 0.06 seconds to deallocate network for instance. [ 1909.433795] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3f54ff6d-85e1-4d00-be13-bfc4a9353992 tempest-ServersNegativeTestMultiTenantJSON-2144504344 tempest-ServersNegativeTestMultiTenantJSON-2144504344-project-member] Lock "e4ebb0f0-ab58-491f-bc4a-7548bbe415d8" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.192s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1909.435095] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "e4ebb0f0-ab58-491f-bc4a-7548bbe415d8" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 117.527s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1909.435308] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: e4ebb0f0-ab58-491f-bc4a-7548bbe415d8] During sync_power_state the instance has a pending task (deleting). Skip. [ 1909.435604] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "e4ebb0f0-ab58-491f-bc4a-7548bbe415d8" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1925.980995] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1932.069451] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1932.071750] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1932.084683] env[67752]: DEBUG oslo_concurrency.lockutils [None req-df260266-bafa-4aec-b55b-68047119aacf tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquiring lock "f2509f7b-1f6b-4942-b591-b7c691fafb26" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1932.634940] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1932.635152] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1932.635265] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1932.635407] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 1935.635955] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1935.636393] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 1935.636393] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 1935.656632] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1935.656801] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1935.656930] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1935.657027] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1935.657155] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1935.657273] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1935.657392] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1935.657509] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1935.657623] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1935.657738] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 1936.635265] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1939.634598] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1939.646074] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1939.646300] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1939.646470] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1939.646627] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 1939.647740] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-45a096e4-f372-4664-aac0-486069336c08 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1939.656330] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56cae299-bdc6-470e-9831-ca9e78171753 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1939.669693] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-463d365d-f7cd-4ebb-8823-daa058cbbcaa {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1939.675692] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cf04a1f7-1292-4a10-ab64-4801eb2ba3ab {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1939.705053] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181017MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 1939.705214] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1939.705404] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1939.773077] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance c1975c17-15a5-4751-bf7f-d5b7e0d2fea1 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1939.773247] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1939.773371] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 7557e3a1-b298-401f-8320-5a48ddda3298 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1939.773517] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 34c75d7e-e63e-4eff-afad-0edd61e023fc actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1939.773644] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d52496eb-f8e3-4d86-8454-f56e9097777e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1939.773759] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 09e003b7-3c4c-4b11-a52d-749acf709068 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1939.773873] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fea66a16-92c3-4c7a-8bff-c6eb59270158 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1939.773988] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f2509f7b-1f6b-4942-b591-b7c691fafb26 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1939.774114] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 369dbb1f-3de0-4ba9-af2b-98167a6b4508 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 1939.774294] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 9 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 1939.774439] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1664MB phys_disk=200GB used_disk=9GB total_vcpus=48 used_vcpus=9 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 1939.791496] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Refreshing inventories for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:804}} [ 1939.804933] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Updating ProviderTree inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:768}} [ 1939.805124] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 1939.815127] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Refreshing aggregate associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, aggregates: None {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:813}} [ 1939.831235] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Refreshing trait associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, traits: COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:825}} [ 1939.928640] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e0a0b720-8d55-4b50-9029-b42fbf0b1236 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1939.935934] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b32fc0f2-1200-4ae5-b076-df4e254317c6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1939.965309] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4806c83-11b4-4e1c-a159-0c97999012d0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1939.972619] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-649ddf4b-3eca-4d0b-ac31-6095f636e06e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1939.985714] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1939.994119] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1940.009963] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 1940.010186] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.305s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1953.927802] env[67752]: WARNING oslo_vmware.rw_handles [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 1953.927802] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 1953.927802] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 1953.927802] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 1953.927802] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 1953.927802] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 1953.927802] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 1953.927802] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 1953.927802] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 1953.927802] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 1953.927802] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 1953.927802] env[67752]: ERROR oslo_vmware.rw_handles [ 1953.928598] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/9189705b-b551-4dc0-afd6-86cbcf65bb3b/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 1953.930440] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 1953.930703] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Copying Virtual Disk [datastore2] vmware_temp/9189705b-b551-4dc0-afd6-86cbcf65bb3b/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/9189705b-b551-4dc0-afd6-86cbcf65bb3b/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 1953.930979] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-79253446-3e7a-4e8b-a72c-e3822199f5a5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1953.939027] env[67752]: DEBUG oslo_vmware.api [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Waiting for the task: (returnval){ [ 1953.939027] env[67752]: value = "task-3199829" [ 1953.939027] env[67752]: _type = "Task" [ 1953.939027] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1953.948664] env[67752]: DEBUG oslo_vmware.api [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Task: {'id': task-3199829, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1954.448506] env[67752]: DEBUG oslo_vmware.exceptions [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 1954.448784] env[67752]: DEBUG oslo_concurrency.lockutils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 1954.449338] env[67752]: ERROR nova.compute.manager [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1954.449338] env[67752]: Faults: ['InvalidArgument'] [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Traceback (most recent call last): [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] yield resources [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] self.driver.spawn(context, instance, image_meta, [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] self._fetch_image_if_missing(context, vi) [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] image_cache(vi, tmp_image_ds_loc) [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] vm_util.copy_virtual_disk( [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] session._wait_for_task(vmdk_copy_task) [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] return self.wait_for_task(task_ref) [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] return evt.wait() [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] result = hub.switch() [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] return self.greenlet.switch() [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] self.f(*self.args, **self.kw) [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] raise exceptions.translate_fault(task_info.error) [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Faults: ['InvalidArgument'] [ 1954.449338] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] [ 1954.450531] env[67752]: INFO nova.compute.manager [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Terminating instance [ 1954.451180] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 1954.451391] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1954.451619] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-41e98186-f357-4968-8c27-de93fca94bde {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1954.453715] env[67752]: DEBUG nova.compute.manager [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1954.453935] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1954.454637] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79472071-52ed-4114-b4d7-b8a3d294f15f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1954.461213] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 1954.461417] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-ed00bb5a-5669-4fa1-b4e5-742ac7a73bcd {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1954.463444] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1954.463618] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 1954.464545] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-84c33ff8-9098-44c3-a824-f3731624fd69 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1954.468984] env[67752]: DEBUG oslo_vmware.api [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for the task: (returnval){ [ 1954.468984] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]528d5565-35d5-7fee-d136-7186124ae4af" [ 1954.468984] env[67752]: _type = "Task" [ 1954.468984] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1954.480067] env[67752]: DEBUG oslo_vmware.api [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]528d5565-35d5-7fee-d136-7186124ae4af, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1954.525516] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 1954.525739] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 1954.525889] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Deleting the datastore file [datastore2] c1975c17-15a5-4751-bf7f-d5b7e0d2fea1 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 1954.526183] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-7d05372b-07c5-46cd-8605-06e1078cb4a4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1954.531899] env[67752]: DEBUG oslo_vmware.api [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Waiting for the task: (returnval){ [ 1954.531899] env[67752]: value = "task-3199831" [ 1954.531899] env[67752]: _type = "Task" [ 1954.531899] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 1954.538933] env[67752]: DEBUG oslo_vmware.api [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Task: {'id': task-3199831, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 1954.979648] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 1954.979943] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Creating directory with path [datastore2] vmware_temp/e5da5c2c-0d19-498b-be64-4b06e96fbdfe/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 1954.980159] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-fc3f54ad-8b95-413e-a146-ad89333f63e8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1954.991702] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Created directory with path [datastore2] vmware_temp/e5da5c2c-0d19-498b-be64-4b06e96fbdfe/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 1954.991913] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Fetch image to [datastore2] vmware_temp/e5da5c2c-0d19-498b-be64-4b06e96fbdfe/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 1954.992110] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/e5da5c2c-0d19-498b-be64-4b06e96fbdfe/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 1954.992832] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c9eb18f9-8dc3-454b-85ca-374efd24c127 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1954.999348] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-868cb8bb-2037-4695-976a-33e66805b97b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1955.008368] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-554cd9fc-ca6b-4621-9563-5c0f6da3fcc8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1955.040727] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-70343ed9-493e-40a5-af5e-77b66a946579 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1955.048369] env[67752]: DEBUG oslo_vmware.api [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Task: {'id': task-3199831, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.078863} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 1955.049696] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 1955.049923] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 1955.050094] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1955.050220] env[67752]: INFO nova.compute.manager [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Took 0.60 seconds to destroy the instance on the hypervisor. [ 1955.051967] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-61ee2fcb-6841-45be-99e0-ece38fdbf06a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1955.053786] env[67752]: DEBUG nova.compute.claims [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 1955.053983] env[67752]: DEBUG oslo_concurrency.lockutils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1955.054214] env[67752]: DEBUG oslo_concurrency.lockutils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1955.077022] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 1955.128139] env[67752]: DEBUG oslo_vmware.rw_handles [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/e5da5c2c-0d19-498b-be64-4b06e96fbdfe/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 1955.191984] env[67752]: DEBUG oslo_vmware.rw_handles [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 1955.192198] env[67752]: DEBUG oslo_vmware.rw_handles [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Closing write handle for https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/e5da5c2c-0d19-498b-be64-4b06e96fbdfe/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 1955.260858] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e51d399b-a0df-4d53-9d82-148265108b2f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1955.268603] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ffdc0fd0-210f-45b4-ac5a-e3780bc78066 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1955.297906] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-425968a6-ca94-40e3-980e-b07aed231139 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1955.304468] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d6c59a1-17a7-4b3a-8b6a-4a07b02bdf67 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1955.316876] env[67752]: DEBUG nova.compute.provider_tree [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1955.325092] env[67752]: DEBUG nova.scheduler.client.report [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1955.340309] env[67752]: DEBUG oslo_concurrency.lockutils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.286s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1955.340826] env[67752]: ERROR nova.compute.manager [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1955.340826] env[67752]: Faults: ['InvalidArgument'] [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Traceback (most recent call last): [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] self.driver.spawn(context, instance, image_meta, [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] self._vmops.spawn(context, instance, image_meta, injected_files, [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] self._fetch_image_if_missing(context, vi) [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] image_cache(vi, tmp_image_ds_loc) [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] vm_util.copy_virtual_disk( [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] session._wait_for_task(vmdk_copy_task) [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] return self.wait_for_task(task_ref) [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] return evt.wait() [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] result = hub.switch() [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] return self.greenlet.switch() [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] self.f(*self.args, **self.kw) [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] raise exceptions.translate_fault(task_info.error) [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Faults: ['InvalidArgument'] [ 1955.340826] env[67752]: ERROR nova.compute.manager [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] [ 1955.342198] env[67752]: DEBUG nova.compute.utils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 1955.342890] env[67752]: DEBUG nova.compute.manager [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Build of instance c1975c17-15a5-4751-bf7f-d5b7e0d2fea1 was re-scheduled: A specified parameter was not correct: fileType [ 1955.342890] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 1955.343270] env[67752]: DEBUG nova.compute.manager [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 1955.343443] env[67752]: DEBUG nova.compute.manager [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 1955.343616] env[67752]: DEBUG nova.compute.manager [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1955.343781] env[67752]: DEBUG nova.network.neutron [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1955.724346] env[67752]: DEBUG nova.network.neutron [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1955.737478] env[67752]: INFO nova.compute.manager [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Took 0.39 seconds to deallocate network for instance. [ 1955.832806] env[67752]: INFO nova.scheduler.client.report [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Deleted allocations for instance c1975c17-15a5-4751-bf7f-d5b7e0d2fea1 [ 1955.858509] env[67752]: DEBUG oslo_concurrency.lockutils [None req-aefe9e66-658d-4f73-bd32-918f7bde0eed tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "c1975c17-15a5-4751-bf7f-d5b7e0d2fea1" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 612.926s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1955.858509] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bcfd9a26-e4d2-4ae9-a941-1ca1dc5e30d4 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "c1975c17-15a5-4751-bf7f-d5b7e0d2fea1" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 417.060s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1955.858509] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bcfd9a26-e4d2-4ae9-a941-1ca1dc5e30d4 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquiring lock "c1975c17-15a5-4751-bf7f-d5b7e0d2fea1-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1955.858509] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bcfd9a26-e4d2-4ae9-a941-1ca1dc5e30d4 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "c1975c17-15a5-4751-bf7f-d5b7e0d2fea1-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1955.858509] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bcfd9a26-e4d2-4ae9-a941-1ca1dc5e30d4 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "c1975c17-15a5-4751-bf7f-d5b7e0d2fea1-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1955.861237] env[67752]: INFO nova.compute.manager [None req-bcfd9a26-e4d2-4ae9-a941-1ca1dc5e30d4 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Terminating instance [ 1955.862120] env[67752]: DEBUG nova.compute.manager [None req-bcfd9a26-e4d2-4ae9-a941-1ca1dc5e30d4 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 1955.863392] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bcfd9a26-e4d2-4ae9-a941-1ca1dc5e30d4 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 1955.863392] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-36a5d216-38ad-4347-a463-bc1362b31a53 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1955.872088] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-174b8f19-ca91-4940-b617-e659e1d2e43c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1955.899454] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-bcfd9a26-e4d2-4ae9-a941-1ca1dc5e30d4 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance c1975c17-15a5-4751-bf7f-d5b7e0d2fea1 could not be found. [ 1955.899667] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bcfd9a26-e4d2-4ae9-a941-1ca1dc5e30d4 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 1955.899847] env[67752]: INFO nova.compute.manager [None req-bcfd9a26-e4d2-4ae9-a941-1ca1dc5e30d4 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Took 0.04 seconds to destroy the instance on the hypervisor. [ 1955.900101] env[67752]: DEBUG oslo.service.loopingcall [None req-bcfd9a26-e4d2-4ae9-a941-1ca1dc5e30d4 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 1955.900324] env[67752]: DEBUG nova.compute.manager [-] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 1955.900416] env[67752]: DEBUG nova.network.neutron [-] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1955.935833] env[67752]: DEBUG nova.network.neutron [-] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1955.943979] env[67752]: INFO nova.compute.manager [-] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] Took 0.04 seconds to deallocate network for instance. [ 1956.039890] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bcfd9a26-e4d2-4ae9-a941-1ca1dc5e30d4 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "c1975c17-15a5-4751-bf7f-d5b7e0d2fea1" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.182s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1956.041018] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "c1975c17-15a5-4751-bf7f-d5b7e0d2fea1" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 164.132s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1956.041212] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: c1975c17-15a5-4751-bf7f-d5b7e0d2fea1] During sync_power_state the instance has a pending task (deleting). Skip. [ 1956.041389] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "c1975c17-15a5-4751-bf7f-d5b7e0d2fea1" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1987.007186] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1991.635640] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1992.635080] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1992.635496] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1993.631973] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1993.653687] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1993.653912] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 1994.635596] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1997.636609] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1997.637028] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 1997.637028] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 1997.656071] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1997.656248] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1997.656373] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1997.656501] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1997.656628] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1997.656752] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1997.656873] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1997.656995] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 1997.657187] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 1997.657688] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 1999.190543] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "e623c04f-674f-46b8-9477-400f8421054e" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1999.190995] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "e623c04f-674f-46b8-9477-400f8421054e" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1999.202295] env[67752]: DEBUG nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1999.226553] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "1a18b0c1-48e6-423d-a43c-6c07ffedbe06" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1999.226784] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "1a18b0c1-48e6-423d-a43c-6c07ffedbe06" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1999.236758] env[67752]: DEBUG nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 1999.258518] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1999.258783] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1999.260212] env[67752]: INFO nova.compute.claims [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1999.287099] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1999.430274] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-755d7bbf-906d-4c69-812b-609ebd5ebf8d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1999.437860] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-40645d8a-427f-4145-b572-64280f41dafa {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1999.468051] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d73fa88a-33c5-4cac-8bfb-e00666630cde {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1999.475160] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-21c974fa-2097-4843-958e-4a16e9366a96 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1999.488871] env[67752]: DEBUG nova.compute.provider_tree [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1999.497574] env[67752]: DEBUG nova.scheduler.client.report [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1999.516317] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.257s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1999.516930] env[67752]: DEBUG nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1999.519204] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.232s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1999.520746] env[67752]: INFO nova.compute.claims [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 1999.557200] env[67752]: DEBUG nova.compute.utils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1999.558974] env[67752]: DEBUG nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1999.559159] env[67752]: DEBUG nova.network.neutron [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1999.567385] env[67752]: DEBUG nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1999.618393] env[67752]: DEBUG nova.policy [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '827cdef2d9b647f79e05f8d23696a090', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd6b1ff6c8aaa49bd968d23a1fe068529', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1999.626784] env[67752]: DEBUG nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1999.651285] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1999.651530] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1999.651693] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1999.651875] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1999.652033] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1999.652189] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1999.652471] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1999.652849] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1999.652849] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1999.652963] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1999.653129] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1999.653970] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-25543634-1cd9-4d39-9c2c-6e06cbb06657 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1999.663384] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79c3ab9a-e314-4d4c-96d8-45c9b563600e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1999.701636] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-087aa073-fe40-4a3d-a289-688726a960cd {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1999.708838] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a039c8f8-af79-4d41-bc08-e8c0f27b9cf0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1999.739553] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c9d6e835-a906-4d90-a516-6d05a27e55e5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1999.746443] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0e294ec1-b9ad-43eb-a145-7d5b53b26c7a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1999.759887] env[67752]: DEBUG nova.compute.provider_tree [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 1999.772359] env[67752]: DEBUG nova.scheduler.client.report [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 1999.787330] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.268s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1999.787804] env[67752]: DEBUG nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 1999.819829] env[67752]: DEBUG nova.compute.utils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 1999.821463] env[67752]: DEBUG nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 1999.821743] env[67752]: DEBUG nova.network.neutron [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 1999.829731] env[67752]: DEBUG nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 1999.892387] env[67752]: DEBUG nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 1999.897436] env[67752]: DEBUG nova.policy [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '827cdef2d9b647f79e05f8d23696a090', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'd6b1ff6c8aaa49bd968d23a1fe068529', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 1999.912243] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 1999.912495] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 1999.912653] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 1999.912835] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 1999.912983] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 1999.913152] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 1999.913363] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 1999.913520] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 1999.913687] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 1999.913847] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 1999.914035] env[67752]: DEBUG nova.virt.hardware [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 1999.915205] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-758ae5ef-a076-409b-88ea-6b037d86e141 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 1999.922629] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b2c0641a-752e-4697-8ff7-4876342c7390 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2000.018073] env[67752]: DEBUG nova.network.neutron [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Successfully created port: 6aa2f43a-04a1-49e4-886b-5054f742aea6 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2001.028973] env[67752]: DEBUG nova.network.neutron [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Successfully created port: 43c44902-7047-4e67-80a2-c6f337205cb6 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2001.635155] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2001.656765] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2001.657039] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2001.657224] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2001.657382] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2001.658916] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-76426072-7786-4ab6-866e-acf9ff6cc1d2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2001.668263] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f9edde8-39d6-49c2-8f27-193648f78c20 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2001.682580] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-62ceb5f3-6f68-4e9a-93f5-fe98c610681d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2001.689806] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e6e4915-842d-4c7b-a5e3-3627d7a39d1d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2001.721668] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181012MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2001.721831] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2001.722054] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2001.746446] env[67752]: DEBUG nova.compute.manager [req-b9e2aca5-72e7-4fc7-b72f-56e4506ed445 req-04b6d03c-ae0b-435f-8737-6556af12db21 service nova] [instance: e623c04f-674f-46b8-9477-400f8421054e] Received event network-vif-plugged-6aa2f43a-04a1-49e4-886b-5054f742aea6 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 2001.746687] env[67752]: DEBUG oslo_concurrency.lockutils [req-b9e2aca5-72e7-4fc7-b72f-56e4506ed445 req-04b6d03c-ae0b-435f-8737-6556af12db21 service nova] Acquiring lock "e623c04f-674f-46b8-9477-400f8421054e-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2001.746900] env[67752]: DEBUG oslo_concurrency.lockutils [req-b9e2aca5-72e7-4fc7-b72f-56e4506ed445 req-04b6d03c-ae0b-435f-8737-6556af12db21 service nova] Lock "e623c04f-674f-46b8-9477-400f8421054e-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2001.747167] env[67752]: DEBUG oslo_concurrency.lockutils [req-b9e2aca5-72e7-4fc7-b72f-56e4506ed445 req-04b6d03c-ae0b-435f-8737-6556af12db21 service nova] Lock "e623c04f-674f-46b8-9477-400f8421054e-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2001.747328] env[67752]: DEBUG nova.compute.manager [req-b9e2aca5-72e7-4fc7-b72f-56e4506ed445 req-04b6d03c-ae0b-435f-8737-6556af12db21 service nova] [instance: e623c04f-674f-46b8-9477-400f8421054e] No waiting events found dispatching network-vif-plugged-6aa2f43a-04a1-49e4-886b-5054f742aea6 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 2001.747492] env[67752]: WARNING nova.compute.manager [req-b9e2aca5-72e7-4fc7-b72f-56e4506ed445 req-04b6d03c-ae0b-435f-8737-6556af12db21 service nova] [instance: e623c04f-674f-46b8-9477-400f8421054e] Received unexpected event network-vif-plugged-6aa2f43a-04a1-49e4-886b-5054f742aea6 for instance with vm_state building and task_state spawning. [ 2001.803384] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2001.803580] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 7557e3a1-b298-401f-8320-5a48ddda3298 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2001.803742] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 34c75d7e-e63e-4eff-afad-0edd61e023fc actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2001.803908] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d52496eb-f8e3-4d86-8454-f56e9097777e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2001.804071] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 09e003b7-3c4c-4b11-a52d-749acf709068 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2001.804300] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fea66a16-92c3-4c7a-8bff-c6eb59270158 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2001.804391] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f2509f7b-1f6b-4942-b591-b7c691fafb26 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2001.804518] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 369dbb1f-3de0-4ba9-af2b-98167a6b4508 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2001.804672] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e623c04f-674f-46b8-9477-400f8421054e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2001.804813] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1a18b0c1-48e6-423d-a43c-6c07ffedbe06 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2001.805106] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2001.805268] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2001.859381] env[67752]: DEBUG nova.network.neutron [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Successfully updated port: 6aa2f43a-04a1-49e4-886b-5054f742aea6 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 2001.871450] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "refresh_cache-e623c04f-674f-46b8-9477-400f8421054e" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2001.871619] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquired lock "refresh_cache-e623c04f-674f-46b8-9477-400f8421054e" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2001.871800] env[67752]: DEBUG nova.network.neutron [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 2001.992403] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0620823e-a5df-4318-81b9-a4bc61c3e168 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2002.000670] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d96a8862-961b-401e-b464-d20fa25c8ad8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2002.032468] env[67752]: DEBUG nova.network.neutron [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 2002.034996] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bf8592c9-ae29-4534-a954-a96c780afa68 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2002.043236] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-573fc56d-ced3-44f5-bbc0-85dc7074bbf2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2002.058390] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2002.069196] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2002.090841] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2002.091111] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.369s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2002.641488] env[67752]: DEBUG nova.network.neutron [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Updating instance_info_cache with network_info: [{"id": "6aa2f43a-04a1-49e4-886b-5054f742aea6", "address": "fa:16:3e:3d:fa:40", "network": {"id": "b761e2cd-93bf-4743-989c-8580aaf2f67a", "bridge": "br-int", "label": "tempest-MultipleCreateTestJSON-1617981042-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d6b1ff6c8aaa49bd968d23a1fe068529", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "e85cbc56-fee0-41f7-bc70-64f31775ce92", "external-id": "nsx-vlan-transportzone-793", "segmentation_id": 793, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap6aa2f43a-04", "ovs_interfaceid": "6aa2f43a-04a1-49e4-886b-5054f742aea6", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2002.655053] env[67752]: DEBUG nova.network.neutron [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Successfully updated port: 43c44902-7047-4e67-80a2-c6f337205cb6 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 2002.662757] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Releasing lock "refresh_cache-e623c04f-674f-46b8-9477-400f8421054e" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2002.663277] env[67752]: DEBUG nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Instance network_info: |[{"id": "6aa2f43a-04a1-49e4-886b-5054f742aea6", "address": "fa:16:3e:3d:fa:40", "network": {"id": "b761e2cd-93bf-4743-989c-8580aaf2f67a", "bridge": "br-int", "label": "tempest-MultipleCreateTestJSON-1617981042-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d6b1ff6c8aaa49bd968d23a1fe068529", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "e85cbc56-fee0-41f7-bc70-64f31775ce92", "external-id": "nsx-vlan-transportzone-793", "segmentation_id": 793, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap6aa2f43a-04", "ovs_interfaceid": "6aa2f43a-04a1-49e4-886b-5054f742aea6", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 2002.663530] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:3d:fa:40', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'e85cbc56-fee0-41f7-bc70-64f31775ce92', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '6aa2f43a-04a1-49e4-886b-5054f742aea6', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 2002.671432] env[67752]: DEBUG oslo.service.loopingcall [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2002.672558] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: e623c04f-674f-46b8-9477-400f8421054e] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 2002.673047] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "refresh_cache-1a18b0c1-48e6-423d-a43c-6c07ffedbe06" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2002.673233] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquired lock "refresh_cache-1a18b0c1-48e6-423d-a43c-6c07ffedbe06" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2002.673324] env[67752]: DEBUG nova.network.neutron [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 2002.674665] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-3d85c5d5-e6d4-4017-a4bc-af96f1c07a12 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2002.698102] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 2002.698102] env[67752]: value = "task-3199832" [ 2002.698102] env[67752]: _type = "Task" [ 2002.698102] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2002.705871] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199832, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2002.740256] env[67752]: DEBUG nova.network.neutron [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 2003.081469] env[67752]: DEBUG nova.network.neutron [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Updating instance_info_cache with network_info: [{"id": "43c44902-7047-4e67-80a2-c6f337205cb6", "address": "fa:16:3e:db:3c:22", "network": {"id": "b761e2cd-93bf-4743-989c-8580aaf2f67a", "bridge": "br-int", "label": "tempest-MultipleCreateTestJSON-1617981042-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d6b1ff6c8aaa49bd968d23a1fe068529", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "e85cbc56-fee0-41f7-bc70-64f31775ce92", "external-id": "nsx-vlan-transportzone-793", "segmentation_id": 793, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap43c44902-70", "ovs_interfaceid": "43c44902-7047-4e67-80a2-c6f337205cb6", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2003.095175] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Releasing lock "refresh_cache-1a18b0c1-48e6-423d-a43c-6c07ffedbe06" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2003.095489] env[67752]: DEBUG nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Instance network_info: |[{"id": "43c44902-7047-4e67-80a2-c6f337205cb6", "address": "fa:16:3e:db:3c:22", "network": {"id": "b761e2cd-93bf-4743-989c-8580aaf2f67a", "bridge": "br-int", "label": "tempest-MultipleCreateTestJSON-1617981042-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d6b1ff6c8aaa49bd968d23a1fe068529", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "e85cbc56-fee0-41f7-bc70-64f31775ce92", "external-id": "nsx-vlan-transportzone-793", "segmentation_id": 793, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap43c44902-70", "ovs_interfaceid": "43c44902-7047-4e67-80a2-c6f337205cb6", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 2003.095909] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:db:3c:22', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'e85cbc56-fee0-41f7-bc70-64f31775ce92', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '43c44902-7047-4e67-80a2-c6f337205cb6', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 2003.104035] env[67752]: DEBUG oslo.service.loopingcall [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2003.104035] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 2003.104219] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-7d29e7ff-1eb7-4b55-843f-fe96b77b23a9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2003.123434] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 2003.123434] env[67752]: value = "task-3199833" [ 2003.123434] env[67752]: _type = "Task" [ 2003.123434] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2003.130649] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199833, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2003.208521] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199832, 'name': CreateVM_Task, 'duration_secs': 0.283648} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2003.208729] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: e623c04f-674f-46b8-9477-400f8421054e] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 2003.209436] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2003.209609] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2003.209932] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 2003.210193] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-7436ace3-0ff7-4d42-83d2-eb378cb7a99a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2003.214533] env[67752]: DEBUG oslo_vmware.api [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for the task: (returnval){ [ 2003.214533] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52394a7c-e38c-d839-b6d0-f0d14b56d96e" [ 2003.214533] env[67752]: _type = "Task" [ 2003.214533] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2003.222177] env[67752]: DEBUG oslo_vmware.api [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52394a7c-e38c-d839-b6d0-f0d14b56d96e, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2003.634183] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199833, 'name': CreateVM_Task, 'duration_secs': 0.27237} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2003.634364] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 2003.634927] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2003.724068] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2003.724341] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 2003.724552] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2003.724761] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2003.725117] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 2003.725363] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-a6376864-c7ea-4d11-bb75-9e2b5ea8259a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2003.731289] env[67752]: DEBUG oslo_vmware.api [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for the task: (returnval){ [ 2003.731289] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]525e8183-e223-bba2-0aa4-c6668b8567e6" [ 2003.731289] env[67752]: _type = "Task" [ 2003.731289] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2003.738181] env[67752]: DEBUG oslo_vmware.api [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]525e8183-e223-bba2-0aa4-c6668b8567e6, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2003.774987] env[67752]: DEBUG nova.compute.manager [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] [instance: e623c04f-674f-46b8-9477-400f8421054e] Received event network-changed-6aa2f43a-04a1-49e4-886b-5054f742aea6 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 2003.774987] env[67752]: DEBUG nova.compute.manager [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] [instance: e623c04f-674f-46b8-9477-400f8421054e] Refreshing instance network info cache due to event network-changed-6aa2f43a-04a1-49e4-886b-5054f742aea6. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 2003.775265] env[67752]: DEBUG oslo_concurrency.lockutils [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] Acquiring lock "refresh_cache-e623c04f-674f-46b8-9477-400f8421054e" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2003.775415] env[67752]: DEBUG oslo_concurrency.lockutils [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] Acquired lock "refresh_cache-e623c04f-674f-46b8-9477-400f8421054e" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2003.775581] env[67752]: DEBUG nova.network.neutron [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] [instance: e623c04f-674f-46b8-9477-400f8421054e] Refreshing network info cache for port 6aa2f43a-04a1-49e4-886b-5054f742aea6 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 2004.058629] env[67752]: DEBUG nova.network.neutron [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] [instance: e623c04f-674f-46b8-9477-400f8421054e] Updated VIF entry in instance network info cache for port 6aa2f43a-04a1-49e4-886b-5054f742aea6. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 2004.059021] env[67752]: DEBUG nova.network.neutron [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] [instance: e623c04f-674f-46b8-9477-400f8421054e] Updating instance_info_cache with network_info: [{"id": "6aa2f43a-04a1-49e4-886b-5054f742aea6", "address": "fa:16:3e:3d:fa:40", "network": {"id": "b761e2cd-93bf-4743-989c-8580aaf2f67a", "bridge": "br-int", "label": "tempest-MultipleCreateTestJSON-1617981042-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d6b1ff6c8aaa49bd968d23a1fe068529", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "e85cbc56-fee0-41f7-bc70-64f31775ce92", "external-id": "nsx-vlan-transportzone-793", "segmentation_id": 793, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap6aa2f43a-04", "ovs_interfaceid": "6aa2f43a-04a1-49e4-886b-5054f742aea6", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2004.068564] env[67752]: DEBUG oslo_concurrency.lockutils [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] Releasing lock "refresh_cache-e623c04f-674f-46b8-9477-400f8421054e" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2004.068806] env[67752]: DEBUG nova.compute.manager [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Received event network-vif-plugged-43c44902-7047-4e67-80a2-c6f337205cb6 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 2004.068999] env[67752]: DEBUG oslo_concurrency.lockutils [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] Acquiring lock "1a18b0c1-48e6-423d-a43c-6c07ffedbe06-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2004.069223] env[67752]: DEBUG oslo_concurrency.lockutils [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] Lock "1a18b0c1-48e6-423d-a43c-6c07ffedbe06-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2004.069392] env[67752]: DEBUG oslo_concurrency.lockutils [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] Lock "1a18b0c1-48e6-423d-a43c-6c07ffedbe06-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2004.069560] env[67752]: DEBUG nova.compute.manager [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] No waiting events found dispatching network-vif-plugged-43c44902-7047-4e67-80a2-c6f337205cb6 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 2004.069727] env[67752]: WARNING nova.compute.manager [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Received unexpected event network-vif-plugged-43c44902-7047-4e67-80a2-c6f337205cb6 for instance with vm_state building and task_state spawning. [ 2004.069890] env[67752]: DEBUG nova.compute.manager [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Received event network-changed-43c44902-7047-4e67-80a2-c6f337205cb6 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 2004.070066] env[67752]: DEBUG nova.compute.manager [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Refreshing instance network info cache due to event network-changed-43c44902-7047-4e67-80a2-c6f337205cb6. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 2004.070253] env[67752]: DEBUG oslo_concurrency.lockutils [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] Acquiring lock "refresh_cache-1a18b0c1-48e6-423d-a43c-6c07ffedbe06" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2004.070393] env[67752]: DEBUG oslo_concurrency.lockutils [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] Acquired lock "refresh_cache-1a18b0c1-48e6-423d-a43c-6c07ffedbe06" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2004.070548] env[67752]: DEBUG nova.network.neutron [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Refreshing network info cache for port 43c44902-7047-4e67-80a2-c6f337205cb6 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 2004.241984] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2004.242544] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 2004.242807] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2004.489550] env[67752]: DEBUG nova.network.neutron [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Updated VIF entry in instance network info cache for port 43c44902-7047-4e67-80a2-c6f337205cb6. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 2004.489917] env[67752]: DEBUG nova.network.neutron [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Updating instance_info_cache with network_info: [{"id": "43c44902-7047-4e67-80a2-c6f337205cb6", "address": "fa:16:3e:db:3c:22", "network": {"id": "b761e2cd-93bf-4743-989c-8580aaf2f67a", "bridge": "br-int", "label": "tempest-MultipleCreateTestJSON-1617981042-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.7", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "d6b1ff6c8aaa49bd968d23a1fe068529", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "e85cbc56-fee0-41f7-bc70-64f31775ce92", "external-id": "nsx-vlan-transportzone-793", "segmentation_id": 793, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap43c44902-70", "ovs_interfaceid": "43c44902-7047-4e67-80a2-c6f337205cb6", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2004.500569] env[67752]: DEBUG oslo_concurrency.lockutils [req-de93a3d0-2ae7-42ad-8fe3-c1caba5f11f6 req-856ce55b-8514-4e2b-82a6-08cd75e0450f service nova] Releasing lock "refresh_cache-1a18b0c1-48e6-423d-a43c-6c07ffedbe06" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2004.606597] env[67752]: WARNING oslo_vmware.rw_handles [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2004.606597] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2004.606597] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2004.606597] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2004.606597] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2004.606597] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 2004.606597] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2004.606597] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2004.606597] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2004.606597] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2004.606597] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2004.606597] env[67752]: ERROR oslo_vmware.rw_handles [ 2004.607447] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/e5da5c2c-0d19-498b-be64-4b06e96fbdfe/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2004.609083] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2004.609328] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Copying Virtual Disk [datastore2] vmware_temp/e5da5c2c-0d19-498b-be64-4b06e96fbdfe/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/e5da5c2c-0d19-498b-be64-4b06e96fbdfe/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2004.609603] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-7fda712e-220f-47ef-9cd8-8596ebed38a5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2004.617522] env[67752]: DEBUG oslo_vmware.api [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for the task: (returnval){ [ 2004.617522] env[67752]: value = "task-3199834" [ 2004.617522] env[67752]: _type = "Task" [ 2004.617522] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2004.624881] env[67752]: DEBUG oslo_vmware.api [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Task: {'id': task-3199834, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2005.127960] env[67752]: DEBUG oslo_vmware.exceptions [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2005.128272] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2005.128831] env[67752]: ERROR nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2005.128831] env[67752]: Faults: ['InvalidArgument'] [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Traceback (most recent call last): [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] yield resources [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] self.driver.spawn(context, instance, image_meta, [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] self._fetch_image_if_missing(context, vi) [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] image_cache(vi, tmp_image_ds_loc) [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] vm_util.copy_virtual_disk( [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] session._wait_for_task(vmdk_copy_task) [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] return self.wait_for_task(task_ref) [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] return evt.wait() [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] result = hub.switch() [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] return self.greenlet.switch() [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] self.f(*self.args, **self.kw) [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] raise exceptions.translate_fault(task_info.error) [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Faults: ['InvalidArgument'] [ 2005.128831] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] [ 2005.130573] env[67752]: INFO nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Terminating instance [ 2005.130806] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2005.131045] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2005.131293] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-ee3ed215-55e8-49e7-b5ef-4ee1659f9fd0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2005.133587] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 2005.133781] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2005.134505] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a0beed10-570a-440b-9578-a71fadd64ccb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2005.140808] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2005.141037] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-eceb662c-cc5e-4e25-b920-8f5c45387490 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2005.143070] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2005.143254] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2005.144182] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-d7d50e45-8f8e-4e38-98d1-9b1dd3ac7e85 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2005.149166] env[67752]: DEBUG oslo_vmware.api [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for the task: (returnval){ [ 2005.149166] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]529f79fa-4dc7-b007-7721-79f80ebbe15c" [ 2005.149166] env[67752]: _type = "Task" [ 2005.149166] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2005.155769] env[67752]: DEBUG oslo_vmware.api [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]529f79fa-4dc7-b007-7721-79f80ebbe15c, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2005.197651] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2005.197867] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2005.198061] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Deleting the datastore file [datastore2] 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2005.198317] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-00c14e59-49aa-4d03-aeef-4a0c910d5fb7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2005.203691] env[67752]: DEBUG oslo_vmware.api [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for the task: (returnval){ [ 2005.203691] env[67752]: value = "task-3199836" [ 2005.203691] env[67752]: _type = "Task" [ 2005.203691] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2005.210864] env[67752]: DEBUG oslo_vmware.api [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Task: {'id': task-3199836, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2005.659898] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2005.660289] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Creating directory with path [datastore2] vmware_temp/001374b4-cea7-4db1-8100-cce2c2640b2e/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2005.660416] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-aff523b7-831b-44d9-bd3a-0180de704d30 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2005.671452] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Created directory with path [datastore2] vmware_temp/001374b4-cea7-4db1-8100-cce2c2640b2e/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2005.671643] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Fetch image to [datastore2] vmware_temp/001374b4-cea7-4db1-8100-cce2c2640b2e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2005.671818] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/001374b4-cea7-4db1-8100-cce2c2640b2e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2005.672642] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-76af13d4-5d58-4ce7-b983-d888b1eab76f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2005.679256] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fa51b018-f0b2-4ba5-968e-a4759116bea3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2005.689379] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fbe9e5fa-e70a-405b-98b5-1bb7b47ec6da {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2005.721649] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b91c88e0-66e0-47b9-858e-1958f51861ec {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2005.729658] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-4b4541db-f440-46b2-9e70-4f16e4dcc16f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2005.731308] env[67752]: DEBUG oslo_vmware.api [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Task: {'id': task-3199836, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.073455} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2005.731558] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2005.731739] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2005.731907] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2005.732099] env[67752]: INFO nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Took 0.60 seconds to destroy the instance on the hypervisor. [ 2005.734346] env[67752]: DEBUG nova.compute.claims [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2005.734528] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2005.734737] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2005.754188] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2005.809185] env[67752]: DEBUG oslo_vmware.rw_handles [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/001374b4-cea7-4db1-8100-cce2c2640b2e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2005.867770] env[67752]: DEBUG oslo_vmware.rw_handles [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2005.867956] env[67752]: DEBUG oslo_vmware.rw_handles [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Closing write handle for https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/001374b4-cea7-4db1-8100-cce2c2640b2e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2005.953769] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ddd98b14-2f91-47fb-a8a8-dc2cd03370fc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2005.962307] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-32f5b7e6-5e5d-4e70-bb74-f345bce4ed8e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2005.992803] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6dfe9a5-2aaf-4d15-a9fb-c2d1c9ac95d7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2005.999567] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-451f3b2d-1ef2-4c4c-9005-8a31391b41de {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2006.011983] env[67752]: DEBUG nova.compute.provider_tree [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2006.020713] env[67752]: DEBUG nova.scheduler.client.report [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2006.034147] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.299s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2006.034633] env[67752]: ERROR nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2006.034633] env[67752]: Faults: ['InvalidArgument'] [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Traceback (most recent call last): [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] self.driver.spawn(context, instance, image_meta, [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] self._fetch_image_if_missing(context, vi) [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] image_cache(vi, tmp_image_ds_loc) [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] vm_util.copy_virtual_disk( [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] session._wait_for_task(vmdk_copy_task) [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] return self.wait_for_task(task_ref) [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] return evt.wait() [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] result = hub.switch() [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] return self.greenlet.switch() [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] self.f(*self.args, **self.kw) [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] raise exceptions.translate_fault(task_info.error) [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Faults: ['InvalidArgument'] [ 2006.034633] env[67752]: ERROR nova.compute.manager [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] [ 2006.035732] env[67752]: DEBUG nova.compute.utils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2006.036719] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Build of instance 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010 was re-scheduled: A specified parameter was not correct: fileType [ 2006.036719] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 2006.037107] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 2006.037287] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 2006.037466] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 2006.037631] env[67752]: DEBUG nova.network.neutron [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2006.342723] env[67752]: DEBUG nova.network.neutron [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2006.353651] env[67752]: INFO nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Took 0.32 seconds to deallocate network for instance. [ 2006.442610] env[67752]: INFO nova.scheduler.client.report [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Deleted allocations for instance 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010 [ 2006.466036] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "0f3f8622-4b01-47d3-bbbb-dcd88c7ca010" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 595.388s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2006.466283] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3b6fbf87-5b81-42cc-aff0-6ea9d5ce826d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "0f3f8622-4b01-47d3-bbbb-dcd88c7ca010" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 399.496s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2006.466361] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3b6fbf87-5b81-42cc-aff0-6ea9d5ce826d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "0f3f8622-4b01-47d3-bbbb-dcd88c7ca010-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2006.466515] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3b6fbf87-5b81-42cc-aff0-6ea9d5ce826d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "0f3f8622-4b01-47d3-bbbb-dcd88c7ca010-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2006.466688] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3b6fbf87-5b81-42cc-aff0-6ea9d5ce826d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "0f3f8622-4b01-47d3-bbbb-dcd88c7ca010-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2006.468777] env[67752]: INFO nova.compute.manager [None req-3b6fbf87-5b81-42cc-aff0-6ea9d5ce826d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Terminating instance [ 2006.470851] env[67752]: DEBUG nova.compute.manager [None req-3b6fbf87-5b81-42cc-aff0-6ea9d5ce826d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 2006.471062] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-3b6fbf87-5b81-42cc-aff0-6ea9d5ce826d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2006.471344] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-bf1fd8f4-4103-4037-8e20-171aa1b74042 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2006.481202] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8d6f36c9-d711-4e46-856e-25ca0e9f2abc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2006.510924] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-3b6fbf87-5b81-42cc-aff0-6ea9d5ce826d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010 could not be found. [ 2006.511175] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-3b6fbf87-5b81-42cc-aff0-6ea9d5ce826d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2006.511395] env[67752]: INFO nova.compute.manager [None req-3b6fbf87-5b81-42cc-aff0-6ea9d5ce826d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2006.511670] env[67752]: DEBUG oslo.service.loopingcall [None req-3b6fbf87-5b81-42cc-aff0-6ea9d5ce826d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2006.511898] env[67752]: DEBUG nova.compute.manager [-] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 2006.512061] env[67752]: DEBUG nova.network.neutron [-] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2006.538777] env[67752]: DEBUG nova.network.neutron [-] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2006.553050] env[67752]: INFO nova.compute.manager [-] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] Took 0.04 seconds to deallocate network for instance. [ 2006.648959] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3b6fbf87-5b81-42cc-aff0-6ea9d5ce826d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "0f3f8622-4b01-47d3-bbbb-dcd88c7ca010" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.183s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2006.650210] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "0f3f8622-4b01-47d3-bbbb-dcd88c7ca010" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 214.741s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2006.650726] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 0f3f8622-4b01-47d3-bbbb-dcd88c7ca010] During sync_power_state the instance has a pending task (deleting). Skip. [ 2006.650931] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "0f3f8622-4b01-47d3-bbbb-dcd88c7ca010" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2031.050793] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f8c5d39b-786a-4b96-8ece-4d4a8e6e493b tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "369dbb1f-3de0-4ba9-af2b-98167a6b4508" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2041.548036] env[67752]: DEBUG oslo_concurrency.lockutils [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "985fe487-26aa-467c-8e65-c0a341fd21ab" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2041.548362] env[67752]: DEBUG oslo_concurrency.lockutils [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "985fe487-26aa-467c-8e65-c0a341fd21ab" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2041.560295] env[67752]: DEBUG nova.compute.manager [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 2041.612008] env[67752]: DEBUG oslo_concurrency.lockutils [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2041.612272] env[67752]: DEBUG oslo_concurrency.lockutils [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2041.613624] env[67752]: INFO nova.compute.claims [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2041.779629] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a903e350-4d92-45cc-ba18-22040cbd0b7b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2041.788616] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0cf3a42e-45ed-44a1-bbc5-0865af04db23 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2041.818840] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-686c8256-5caf-4674-b160-f39fd8d0247a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2041.825902] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ed39ee8c-a22b-47dc-bfae-3a5bff8b88af {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2041.838702] env[67752]: DEBUG nova.compute.provider_tree [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2041.848196] env[67752]: DEBUG nova.scheduler.client.report [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2041.864087] env[67752]: DEBUG oslo_concurrency.lockutils [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.252s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2041.864577] env[67752]: DEBUG nova.compute.manager [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 2041.897332] env[67752]: DEBUG nova.compute.utils [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2041.898867] env[67752]: DEBUG nova.compute.manager [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 2041.899248] env[67752]: DEBUG nova.network.neutron [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 2041.907835] env[67752]: DEBUG nova.compute.manager [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 2041.954843] env[67752]: DEBUG nova.policy [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7ceddc353fd444808a8e3fa9a98925c8', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '17776dc4e25145179ebfbdfd4a9639f8', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 2041.974716] env[67752]: DEBUG nova.compute.manager [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 2042.000437] env[67752]: DEBUG nova.virt.hardware [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 2042.000686] env[67752]: DEBUG nova.virt.hardware [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 2042.000843] env[67752]: DEBUG nova.virt.hardware [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2042.001041] env[67752]: DEBUG nova.virt.hardware [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 2042.001197] env[67752]: DEBUG nova.virt.hardware [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2042.001351] env[67752]: DEBUG nova.virt.hardware [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 2042.001559] env[67752]: DEBUG nova.virt.hardware [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 2042.001723] env[67752]: DEBUG nova.virt.hardware [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 2042.001894] env[67752]: DEBUG nova.virt.hardware [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 2042.002075] env[67752]: DEBUG nova.virt.hardware [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 2042.002262] env[67752]: DEBUG nova.virt.hardware [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 2042.003120] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cf5bf667-5d8f-4a4b-8476-a173e63177eb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2042.011213] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7a27392-f958-4cee-9c86-540235696406 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2042.382549] env[67752]: DEBUG nova.network.neutron [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Successfully created port: 7616fa16-a3d4-4016-8113-0eefdea505aa {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2043.101519] env[67752]: DEBUG nova.compute.manager [req-9327addc-d7c8-4021-bf14-995b01bdc729 req-064eb6d3-cea4-480f-a5e6-b1e3e2362099 service nova] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Received event network-vif-plugged-7616fa16-a3d4-4016-8113-0eefdea505aa {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 2043.101833] env[67752]: DEBUG oslo_concurrency.lockutils [req-9327addc-d7c8-4021-bf14-995b01bdc729 req-064eb6d3-cea4-480f-a5e6-b1e3e2362099 service nova] Acquiring lock "985fe487-26aa-467c-8e65-c0a341fd21ab-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2043.102209] env[67752]: DEBUG oslo_concurrency.lockutils [req-9327addc-d7c8-4021-bf14-995b01bdc729 req-064eb6d3-cea4-480f-a5e6-b1e3e2362099 service nova] Lock "985fe487-26aa-467c-8e65-c0a341fd21ab-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2043.102445] env[67752]: DEBUG oslo_concurrency.lockutils [req-9327addc-d7c8-4021-bf14-995b01bdc729 req-064eb6d3-cea4-480f-a5e6-b1e3e2362099 service nova] Lock "985fe487-26aa-467c-8e65-c0a341fd21ab-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2043.102671] env[67752]: DEBUG nova.compute.manager [req-9327addc-d7c8-4021-bf14-995b01bdc729 req-064eb6d3-cea4-480f-a5e6-b1e3e2362099 service nova] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] No waiting events found dispatching network-vif-plugged-7616fa16-a3d4-4016-8113-0eefdea505aa {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 2043.102852] env[67752]: WARNING nova.compute.manager [req-9327addc-d7c8-4021-bf14-995b01bdc729 req-064eb6d3-cea4-480f-a5e6-b1e3e2362099 service nova] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Received unexpected event network-vif-plugged-7616fa16-a3d4-4016-8113-0eefdea505aa for instance with vm_state building and task_state spawning. [ 2043.135561] env[67752]: DEBUG nova.network.neutron [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Successfully updated port: 7616fa16-a3d4-4016-8113-0eefdea505aa {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 2043.149583] env[67752]: DEBUG oslo_concurrency.lockutils [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "refresh_cache-985fe487-26aa-467c-8e65-c0a341fd21ab" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2043.149732] env[67752]: DEBUG oslo_concurrency.lockutils [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquired lock "refresh_cache-985fe487-26aa-467c-8e65-c0a341fd21ab" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2043.149882] env[67752]: DEBUG nova.network.neutron [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 2043.192329] env[67752]: DEBUG nova.network.neutron [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 2043.439428] env[67752]: DEBUG nova.network.neutron [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Updating instance_info_cache with network_info: [{"id": "7616fa16-a3d4-4016-8113-0eefdea505aa", "address": "fa:16:3e:0e:ce:9a", "network": {"id": "0fbb58d7-f336-4105-b40d-14fe23cb5231", "bridge": "br-int", "label": "tempest-ServersTestJSON-455316140-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "17776dc4e25145179ebfbdfd4a9639f8", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "92cdccfd-4b10-4024-b724-5f22792dd4de", "external-id": "nsx-vlan-transportzone-902", "segmentation_id": 902, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap7616fa16-a3", "ovs_interfaceid": "7616fa16-a3d4-4016-8113-0eefdea505aa", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2043.449533] env[67752]: DEBUG oslo_concurrency.lockutils [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Releasing lock "refresh_cache-985fe487-26aa-467c-8e65-c0a341fd21ab" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2043.449816] env[67752]: DEBUG nova.compute.manager [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Instance network_info: |[{"id": "7616fa16-a3d4-4016-8113-0eefdea505aa", "address": "fa:16:3e:0e:ce:9a", "network": {"id": "0fbb58d7-f336-4105-b40d-14fe23cb5231", "bridge": "br-int", "label": "tempest-ServersTestJSON-455316140-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "17776dc4e25145179ebfbdfd4a9639f8", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "92cdccfd-4b10-4024-b724-5f22792dd4de", "external-id": "nsx-vlan-transportzone-902", "segmentation_id": 902, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap7616fa16-a3", "ovs_interfaceid": "7616fa16-a3d4-4016-8113-0eefdea505aa", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 2043.450225] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:0e:ce:9a', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '92cdccfd-4b10-4024-b724-5f22792dd4de', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '7616fa16-a3d4-4016-8113-0eefdea505aa', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 2043.457590] env[67752]: DEBUG oslo.service.loopingcall [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2043.458040] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 2043.458276] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-65b5eaa4-04e9-4f97-ac98-50d79f12de13 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2043.480439] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 2043.480439] env[67752]: value = "task-3199837" [ 2043.480439] env[67752]: _type = "Task" [ 2043.480439] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2043.487829] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199837, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2043.991261] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199837, 'name': CreateVM_Task, 'duration_secs': 0.307901} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2043.991471] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 2043.992114] env[67752]: DEBUG oslo_concurrency.lockutils [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2043.992284] env[67752]: DEBUG oslo_concurrency.lockutils [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2043.992607] env[67752]: DEBUG oslo_concurrency.lockutils [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 2043.992851] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-0a1c0891-bc06-4212-b2e0-9a860d00dd6b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2043.997335] env[67752]: DEBUG oslo_vmware.api [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Waiting for the task: (returnval){ [ 2043.997335] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5224f3a3-ab82-4770-9f0b-2b9b2849735b" [ 2043.997335] env[67752]: _type = "Task" [ 2043.997335] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2044.004453] env[67752]: DEBUG oslo_vmware.api [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5224f3a3-ab82-4770-9f0b-2b9b2849735b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2044.507402] env[67752]: DEBUG oslo_concurrency.lockutils [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2044.507711] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 2044.507862] env[67752]: DEBUG oslo_concurrency.lockutils [None req-628362b2-d737-4de7-aa9a-061e1393fd68 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2045.127631] env[67752]: DEBUG nova.compute.manager [req-b3798d46-5603-4ad0-b53e-ba077b14c22a req-19dbcea9-4849-4eb0-978c-b45d21ea3957 service nova] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Received event network-changed-7616fa16-a3d4-4016-8113-0eefdea505aa {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 2045.127848] env[67752]: DEBUG nova.compute.manager [req-b3798d46-5603-4ad0-b53e-ba077b14c22a req-19dbcea9-4849-4eb0-978c-b45d21ea3957 service nova] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Refreshing instance network info cache due to event network-changed-7616fa16-a3d4-4016-8113-0eefdea505aa. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 2045.128086] env[67752]: DEBUG oslo_concurrency.lockutils [req-b3798d46-5603-4ad0-b53e-ba077b14c22a req-19dbcea9-4849-4eb0-978c-b45d21ea3957 service nova] Acquiring lock "refresh_cache-985fe487-26aa-467c-8e65-c0a341fd21ab" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2045.128238] env[67752]: DEBUG oslo_concurrency.lockutils [req-b3798d46-5603-4ad0-b53e-ba077b14c22a req-19dbcea9-4849-4eb0-978c-b45d21ea3957 service nova] Acquired lock "refresh_cache-985fe487-26aa-467c-8e65-c0a341fd21ab" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2045.128423] env[67752]: DEBUG nova.network.neutron [req-b3798d46-5603-4ad0-b53e-ba077b14c22a req-19dbcea9-4849-4eb0-978c-b45d21ea3957 service nova] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Refreshing network info cache for port 7616fa16-a3d4-4016-8113-0eefdea505aa {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 2045.449339] env[67752]: DEBUG nova.network.neutron [req-b3798d46-5603-4ad0-b53e-ba077b14c22a req-19dbcea9-4849-4eb0-978c-b45d21ea3957 service nova] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Updated VIF entry in instance network info cache for port 7616fa16-a3d4-4016-8113-0eefdea505aa. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 2045.449688] env[67752]: DEBUG nova.network.neutron [req-b3798d46-5603-4ad0-b53e-ba077b14c22a req-19dbcea9-4849-4eb0-978c-b45d21ea3957 service nova] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Updating instance_info_cache with network_info: [{"id": "7616fa16-a3d4-4016-8113-0eefdea505aa", "address": "fa:16:3e:0e:ce:9a", "network": {"id": "0fbb58d7-f336-4105-b40d-14fe23cb5231", "bridge": "br-int", "label": "tempest-ServersTestJSON-455316140-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.6", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "17776dc4e25145179ebfbdfd4a9639f8", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "92cdccfd-4b10-4024-b724-5f22792dd4de", "external-id": "nsx-vlan-transportzone-902", "segmentation_id": 902, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap7616fa16-a3", "ovs_interfaceid": "7616fa16-a3d4-4016-8113-0eefdea505aa", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2045.458799] env[67752]: DEBUG oslo_concurrency.lockutils [req-b3798d46-5603-4ad0-b53e-ba077b14c22a req-19dbcea9-4849-4eb0-978c-b45d21ea3957 service nova] Releasing lock "refresh_cache-985fe487-26aa-467c-8e65-c0a341fd21ab" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2046.635079] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2046.635384] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Cleaning up deleted instances with incomplete migration {{(pid=67752) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11257}} [ 2047.639937] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2052.634650] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2052.635568] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2053.635651] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2054.627294] env[67752]: WARNING oslo_vmware.rw_handles [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2054.627294] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2054.627294] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2054.627294] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2054.627294] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2054.627294] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 2054.627294] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2054.627294] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2054.627294] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2054.627294] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2054.627294] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2054.627294] env[67752]: ERROR oslo_vmware.rw_handles [ 2054.627995] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/001374b4-cea7-4db1-8100-cce2c2640b2e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2054.629935] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2054.630355] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Copying Virtual Disk [datastore2] vmware_temp/001374b4-cea7-4db1-8100-cce2c2640b2e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/001374b4-cea7-4db1-8100-cce2c2640b2e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2054.630665] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-afdca10e-c4d2-4c74-95af-603a6fbf4d1f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2054.633924] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2054.639382] env[67752]: DEBUG oslo_vmware.api [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for the task: (returnval){ [ 2054.639382] env[67752]: value = "task-3199838" [ 2054.639382] env[67752]: _type = "Task" [ 2054.639382] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2054.647448] env[67752]: DEBUG oslo_vmware.api [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Task: {'id': task-3199838, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2055.150249] env[67752]: DEBUG oslo_vmware.exceptions [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2055.150595] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2055.151199] env[67752]: ERROR nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2055.151199] env[67752]: Faults: ['InvalidArgument'] [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Traceback (most recent call last): [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] yield resources [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] self.driver.spawn(context, instance, image_meta, [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] self._fetch_image_if_missing(context, vi) [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] image_cache(vi, tmp_image_ds_loc) [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] vm_util.copy_virtual_disk( [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] session._wait_for_task(vmdk_copy_task) [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] return self.wait_for_task(task_ref) [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] return evt.wait() [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] result = hub.switch() [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] return self.greenlet.switch() [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] self.f(*self.args, **self.kw) [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] raise exceptions.translate_fault(task_info.error) [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Faults: ['InvalidArgument'] [ 2055.151199] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] [ 2055.152229] env[67752]: INFO nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Terminating instance [ 2055.153265] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2055.153478] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2055.153718] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-df5ead5e-8cd2-4ea8-9e14-e8ce8eb5f766 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2055.155936] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 2055.156150] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2055.156964] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e046f015-5a3e-4633-b9ab-1773e7c8c079 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2055.164588] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2055.164799] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-b27a7467-1c62-4d94-95cc-a32e9a20a2d6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2055.166967] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2055.167171] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2055.168081] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-4d2dba29-9776-4901-b9f8-85635645107a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2055.172551] env[67752]: DEBUG oslo_vmware.api [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Waiting for the task: (returnval){ [ 2055.172551] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52dfa89b-7f2b-3faf-baa3-00ed6201545b" [ 2055.172551] env[67752]: _type = "Task" [ 2055.172551] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2055.181450] env[67752]: DEBUG oslo_vmware.api [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52dfa89b-7f2b-3faf-baa3-00ed6201545b, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2055.229066] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2055.229416] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2055.229565] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Deleting the datastore file [datastore2] 7557e3a1-b298-401f-8320-5a48ddda3298 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2055.230393] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-0cf1d2ae-694c-4240-b1b6-d192b33da57d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2055.236521] env[67752]: DEBUG oslo_vmware.api [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for the task: (returnval){ [ 2055.236521] env[67752]: value = "task-3199840" [ 2055.236521] env[67752]: _type = "Task" [ 2055.236521] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2055.244727] env[67752]: DEBUG oslo_vmware.api [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Task: {'id': task-3199840, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2055.635254] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2055.635447] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 2055.685839] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2055.689181] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Creating directory with path [datastore2] vmware_temp/db61bf1f-f519-4eb4-80d8-df6ce68603fa/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2055.689462] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-e962bc3f-1783-4151-baad-c0fbb2cc0b04 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2055.703286] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Created directory with path [datastore2] vmware_temp/db61bf1f-f519-4eb4-80d8-df6ce68603fa/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2055.703500] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Fetch image to [datastore2] vmware_temp/db61bf1f-f519-4eb4-80d8-df6ce68603fa/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2055.703674] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/db61bf1f-f519-4eb4-80d8-df6ce68603fa/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2055.704960] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b9ae440-3642-41d5-80ff-da1cb4272785 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2055.713084] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-444c6aef-b379-40f2-98ef-dcc19dba1ec0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2055.722269] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a57f16ff-1f9a-4c6b-8317-6ad7a748819a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2055.756292] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-468e05a4-44d5-45c5-964c-f3c8c94864c7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2055.763407] env[67752]: DEBUG oslo_vmware.api [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Task: {'id': task-3199840, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.078199} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2055.764893] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2055.765104] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2055.765286] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2055.765466] env[67752]: INFO nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Took 0.61 seconds to destroy the instance on the hypervisor. [ 2055.767601] env[67752]: DEBUG nova.compute.claims [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2055.767770] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2055.767986] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2055.770703] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-a9f869de-95f8-4afa-87ca-f871c0ff5251 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2055.792178] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2055.855240] env[67752]: DEBUG oslo_vmware.rw_handles [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/db61bf1f-f519-4eb4-80d8-df6ce68603fa/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2055.921616] env[67752]: DEBUG oslo_concurrency.lockutils [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Acquiring lock "da4910c9-6710-436c-8112-1b8030286623" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2055.921880] env[67752]: DEBUG oslo_concurrency.lockutils [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Lock "da4910c9-6710-436c-8112-1b8030286623" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2055.922335] env[67752]: DEBUG oslo_vmware.rw_handles [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2055.922502] env[67752]: DEBUG oslo_vmware.rw_handles [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/db61bf1f-f519-4eb4-80d8-df6ce68603fa/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2056.096251] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f77ca62b-2d8f-4dd8-8b48-0f93dfeb9085 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2056.103310] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-16f82f2f-371b-40e2-bbc0-26d4f0d6f227 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2056.132524] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e8b101b0-905f-4af3-8f86-1d21fac5a779 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2056.139317] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-300ceff5-5f5f-4015-b782-2b9d2d784cdb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2056.153130] env[67752]: DEBUG nova.compute.provider_tree [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2056.164443] env[67752]: DEBUG nova.scheduler.client.report [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2056.181407] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.413s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2056.182059] env[67752]: ERROR nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2056.182059] env[67752]: Faults: ['InvalidArgument'] [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Traceback (most recent call last): [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] self.driver.spawn(context, instance, image_meta, [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] self._fetch_image_if_missing(context, vi) [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] image_cache(vi, tmp_image_ds_loc) [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] vm_util.copy_virtual_disk( [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] session._wait_for_task(vmdk_copy_task) [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] return self.wait_for_task(task_ref) [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] return evt.wait() [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] result = hub.switch() [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] return self.greenlet.switch() [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] self.f(*self.args, **self.kw) [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] raise exceptions.translate_fault(task_info.error) [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Faults: ['InvalidArgument'] [ 2056.182059] env[67752]: ERROR nova.compute.manager [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] [ 2056.183263] env[67752]: DEBUG nova.compute.utils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2056.184777] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Build of instance 7557e3a1-b298-401f-8320-5a48ddda3298 was re-scheduled: A specified parameter was not correct: fileType [ 2056.184777] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 2056.185366] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 2056.185586] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 2056.185795] env[67752]: DEBUG nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 2056.186013] env[67752]: DEBUG nova.network.neutron [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2056.585690] env[67752]: DEBUG nova.network.neutron [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2056.603131] env[67752]: INFO nova.compute.manager [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Took 0.41 seconds to deallocate network for instance. [ 2056.708358] env[67752]: INFO nova.scheduler.client.report [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Deleted allocations for instance 7557e3a1-b298-401f-8320-5a48ddda3298 [ 2056.734057] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f22af375-ec53-4f60-8b23-1b1843f8774d tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "7557e3a1-b298-401f-8320-5a48ddda3298" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 645.626s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2056.735244] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fd57df97-ba6f-4c42-bc2b-3195c689e06c tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "7557e3a1-b298-401f-8320-5a48ddda3298" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 449.827s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2056.735497] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fd57df97-ba6f-4c42-bc2b-3195c689e06c tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "7557e3a1-b298-401f-8320-5a48ddda3298-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2056.735755] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fd57df97-ba6f-4c42-bc2b-3195c689e06c tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "7557e3a1-b298-401f-8320-5a48ddda3298-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2056.735937] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fd57df97-ba6f-4c42-bc2b-3195c689e06c tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "7557e3a1-b298-401f-8320-5a48ddda3298-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2056.738306] env[67752]: INFO nova.compute.manager [None req-fd57df97-ba6f-4c42-bc2b-3195c689e06c tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Terminating instance [ 2056.741519] env[67752]: DEBUG nova.compute.manager [None req-fd57df97-ba6f-4c42-bc2b-3195c689e06c tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 2056.742321] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-fd57df97-ba6f-4c42-bc2b-3195c689e06c tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2056.742321] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-e3b4b29f-78ee-4d07-a014-e42a054b4e42 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2056.746614] env[67752]: DEBUG nova.compute.manager [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: da4910c9-6710-436c-8112-1b8030286623] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 2056.756013] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-57222a54-e2a9-4e30-afbc-5082ee3447d1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2056.781290] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-fd57df97-ba6f-4c42-bc2b-3195c689e06c tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 7557e3a1-b298-401f-8320-5a48ddda3298 could not be found. [ 2056.781492] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-fd57df97-ba6f-4c42-bc2b-3195c689e06c tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2056.781665] env[67752]: INFO nova.compute.manager [None req-fd57df97-ba6f-4c42-bc2b-3195c689e06c tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2056.781898] env[67752]: DEBUG oslo.service.loopingcall [None req-fd57df97-ba6f-4c42-bc2b-3195c689e06c tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2056.784070] env[67752]: DEBUG nova.compute.manager [-] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 2056.784178] env[67752]: DEBUG nova.network.neutron [-] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2056.798853] env[67752]: DEBUG oslo_concurrency.lockutils [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2056.799146] env[67752]: DEBUG oslo_concurrency.lockutils [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2056.800709] env[67752]: INFO nova.compute.claims [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: da4910c9-6710-436c-8112-1b8030286623] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2056.812938] env[67752]: DEBUG nova.network.neutron [-] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2056.828896] env[67752]: INFO nova.compute.manager [-] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] Took 0.04 seconds to deallocate network for instance. [ 2056.915371] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fd57df97-ba6f-4c42-bc2b-3195c689e06c tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "7557e3a1-b298-401f-8320-5a48ddda3298" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.180s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2056.916283] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "7557e3a1-b298-401f-8320-5a48ddda3298" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 265.007s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2056.916467] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 7557e3a1-b298-401f-8320-5a48ddda3298] During sync_power_state the instance has a pending task (deleting). Skip. [ 2056.916648] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "7557e3a1-b298-401f-8320-5a48ddda3298" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2056.968061] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e269ae3e-0fd3-4753-873b-3a6107a5a5ff {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2056.976516] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-72196ee6-ef27-4cf6-ad4f-12dd6390ea52 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2057.006796] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f7cff25-3b56-4300-9574-015ee2ee1dd6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2057.013795] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9d7db5ac-1e5f-4520-b1b9-a0ecac651458 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2057.026803] env[67752]: DEBUG nova.compute.provider_tree [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2057.036121] env[67752]: DEBUG nova.scheduler.client.report [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2057.051386] env[67752]: DEBUG oslo_concurrency.lockutils [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.252s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2057.051931] env[67752]: DEBUG nova.compute.manager [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: da4910c9-6710-436c-8112-1b8030286623] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 2057.081938] env[67752]: DEBUG nova.compute.utils [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2057.083583] env[67752]: DEBUG nova.compute.manager [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: da4910c9-6710-436c-8112-1b8030286623] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 2057.083753] env[67752]: DEBUG nova.network.neutron [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: da4910c9-6710-436c-8112-1b8030286623] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 2057.091510] env[67752]: DEBUG nova.compute.manager [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: da4910c9-6710-436c-8112-1b8030286623] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 2057.139287] env[67752]: DEBUG nova.policy [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '40a6c9b19bc14ed0b4e1b5b3b031c692', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f7d3fb6872294f31bd9fa2b48f19808f', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 2057.149849] env[67752]: DEBUG nova.compute.manager [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: da4910c9-6710-436c-8112-1b8030286623] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 2057.175318] env[67752]: DEBUG nova.virt.hardware [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 2057.175568] env[67752]: DEBUG nova.virt.hardware [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 2057.175727] env[67752]: DEBUG nova.virt.hardware [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2057.175911] env[67752]: DEBUG nova.virt.hardware [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 2057.176075] env[67752]: DEBUG nova.virt.hardware [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2057.176281] env[67752]: DEBUG nova.virt.hardware [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 2057.176520] env[67752]: DEBUG nova.virt.hardware [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 2057.176688] env[67752]: DEBUG nova.virt.hardware [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 2057.176857] env[67752]: DEBUG nova.virt.hardware [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 2057.177040] env[67752]: DEBUG nova.virt.hardware [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 2057.177240] env[67752]: DEBUG nova.virt.hardware [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 2057.178121] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-780e74c4-0e1b-4537-8833-20452908d92f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2057.185759] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f2a9ae05-ac0b-415a-b888-536ef513a815 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2057.512530] env[67752]: DEBUG nova.network.neutron [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: da4910c9-6710-436c-8112-1b8030286623] Successfully created port: 95e236fd-5101-4990-8a09-f52ba45d2a35 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2058.213997] env[67752]: DEBUG nova.network.neutron [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: da4910c9-6710-436c-8112-1b8030286623] Successfully updated port: 95e236fd-5101-4990-8a09-f52ba45d2a35 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 2058.224556] env[67752]: DEBUG oslo_concurrency.lockutils [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Acquiring lock "refresh_cache-da4910c9-6710-436c-8112-1b8030286623" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2058.224710] env[67752]: DEBUG oslo_concurrency.lockutils [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Acquired lock "refresh_cache-da4910c9-6710-436c-8112-1b8030286623" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2058.224862] env[67752]: DEBUG nova.network.neutron [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: da4910c9-6710-436c-8112-1b8030286623] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 2058.265627] env[67752]: DEBUG nova.network.neutron [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: da4910c9-6710-436c-8112-1b8030286623] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 2058.428042] env[67752]: DEBUG nova.network.neutron [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: da4910c9-6710-436c-8112-1b8030286623] Updating instance_info_cache with network_info: [{"id": "95e236fd-5101-4990-8a09-f52ba45d2a35", "address": "fa:16:3e:dc:2c:a8", "network": {"id": "bef55da3-1050-4c26-9415-aef048e94b8c", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-327762555-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "f7d3fb6872294f31bd9fa2b48f19808f", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "d6e940e5-e083-4238-973e-f1b4e2a3a5c7", "external-id": "nsx-vlan-transportzone-64", "segmentation_id": 64, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap95e236fd-51", "ovs_interfaceid": "95e236fd-5101-4990-8a09-f52ba45d2a35", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2058.456090] env[67752]: DEBUG oslo_concurrency.lockutils [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Releasing lock "refresh_cache-da4910c9-6710-436c-8112-1b8030286623" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2058.456513] env[67752]: DEBUG nova.compute.manager [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: da4910c9-6710-436c-8112-1b8030286623] Instance network_info: |[{"id": "95e236fd-5101-4990-8a09-f52ba45d2a35", "address": "fa:16:3e:dc:2c:a8", "network": {"id": "bef55da3-1050-4c26-9415-aef048e94b8c", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-327762555-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "f7d3fb6872294f31bd9fa2b48f19808f", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "d6e940e5-e083-4238-973e-f1b4e2a3a5c7", "external-id": "nsx-vlan-transportzone-64", "segmentation_id": 64, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap95e236fd-51", "ovs_interfaceid": "95e236fd-5101-4990-8a09-f52ba45d2a35", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 2058.457260] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: da4910c9-6710-436c-8112-1b8030286623] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:dc:2c:a8', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'd6e940e5-e083-4238-973e-f1b4e2a3a5c7', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '95e236fd-5101-4990-8a09-f52ba45d2a35', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 2058.464740] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Creating folder: Project (f7d3fb6872294f31bd9fa2b48f19808f). Parent ref: group-v639722. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 2058.465573] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-29b1f26a-e0fb-44cf-888c-303bddf4dace {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2058.480265] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Created folder: Project (f7d3fb6872294f31bd9fa2b48f19808f) in parent group-v639722. [ 2058.480454] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Creating folder: Instances. Parent ref: group-v639838. {{(pid=67752) create_folder /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1589}} [ 2058.480679] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateFolder with opID=oslo.vmware-258bb064-928d-4068-afda-95d3ca7dcdd2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2058.490040] env[67752]: INFO nova.virt.vmwareapi.vm_util [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Created folder: Instances in parent group-v639838. [ 2058.490260] env[67752]: DEBUG oslo.service.loopingcall [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2058.490437] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: da4910c9-6710-436c-8112-1b8030286623] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 2058.490621] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-0cd584d4-d3f9-4af3-b978-9d3adff6fcaa {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2058.508878] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 2058.508878] env[67752]: value = "task-3199843" [ 2058.508878] env[67752]: _type = "Task" [ 2058.508878] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2058.516891] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199843, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2058.634689] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2058.634873] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 2058.634995] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 2058.680894] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2058.681105] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2058.681170] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2058.681259] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2058.681390] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2058.681516] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2058.681638] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: e623c04f-674f-46b8-9477-400f8421054e] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2058.681760] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2058.681881] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2058.681998] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: da4910c9-6710-436c-8112-1b8030286623] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2058.682131] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 2058.682622] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2058.682819] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2058.682956] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Cleaning up deleted instances {{(pid=67752) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11219}} [ 2058.692983] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] There are 0 instances to clean {{(pid=67752) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11228}} [ 2058.728092] env[67752]: DEBUG nova.compute.manager [req-4e2afbd0-00a5-4949-9760-1556dd09882a req-601548a4-9e6f-4387-8654-ea39e7f3229a service nova] [instance: da4910c9-6710-436c-8112-1b8030286623] Received event network-vif-plugged-95e236fd-5101-4990-8a09-f52ba45d2a35 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 2058.728383] env[67752]: DEBUG oslo_concurrency.lockutils [req-4e2afbd0-00a5-4949-9760-1556dd09882a req-601548a4-9e6f-4387-8654-ea39e7f3229a service nova] Acquiring lock "da4910c9-6710-436c-8112-1b8030286623-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2058.728522] env[67752]: DEBUG oslo_concurrency.lockutils [req-4e2afbd0-00a5-4949-9760-1556dd09882a req-601548a4-9e6f-4387-8654-ea39e7f3229a service nova] Lock "da4910c9-6710-436c-8112-1b8030286623-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2058.728692] env[67752]: DEBUG oslo_concurrency.lockutils [req-4e2afbd0-00a5-4949-9760-1556dd09882a req-601548a4-9e6f-4387-8654-ea39e7f3229a service nova] Lock "da4910c9-6710-436c-8112-1b8030286623-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2058.728859] env[67752]: DEBUG nova.compute.manager [req-4e2afbd0-00a5-4949-9760-1556dd09882a req-601548a4-9e6f-4387-8654-ea39e7f3229a service nova] [instance: da4910c9-6710-436c-8112-1b8030286623] No waiting events found dispatching network-vif-plugged-95e236fd-5101-4990-8a09-f52ba45d2a35 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 2058.729026] env[67752]: WARNING nova.compute.manager [req-4e2afbd0-00a5-4949-9760-1556dd09882a req-601548a4-9e6f-4387-8654-ea39e7f3229a service nova] [instance: da4910c9-6710-436c-8112-1b8030286623] Received unexpected event network-vif-plugged-95e236fd-5101-4990-8a09-f52ba45d2a35 for instance with vm_state building and task_state spawning. [ 2058.729191] env[67752]: DEBUG nova.compute.manager [req-4e2afbd0-00a5-4949-9760-1556dd09882a req-601548a4-9e6f-4387-8654-ea39e7f3229a service nova] [instance: da4910c9-6710-436c-8112-1b8030286623] Received event network-changed-95e236fd-5101-4990-8a09-f52ba45d2a35 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 2058.729348] env[67752]: DEBUG nova.compute.manager [req-4e2afbd0-00a5-4949-9760-1556dd09882a req-601548a4-9e6f-4387-8654-ea39e7f3229a service nova] [instance: da4910c9-6710-436c-8112-1b8030286623] Refreshing instance network info cache due to event network-changed-95e236fd-5101-4990-8a09-f52ba45d2a35. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 2058.729529] env[67752]: DEBUG oslo_concurrency.lockutils [req-4e2afbd0-00a5-4949-9760-1556dd09882a req-601548a4-9e6f-4387-8654-ea39e7f3229a service nova] Acquiring lock "refresh_cache-da4910c9-6710-436c-8112-1b8030286623" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2058.729657] env[67752]: DEBUG oslo_concurrency.lockutils [req-4e2afbd0-00a5-4949-9760-1556dd09882a req-601548a4-9e6f-4387-8654-ea39e7f3229a service nova] Acquired lock "refresh_cache-da4910c9-6710-436c-8112-1b8030286623" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2058.729904] env[67752]: DEBUG nova.network.neutron [req-4e2afbd0-00a5-4949-9760-1556dd09882a req-601548a4-9e6f-4387-8654-ea39e7f3229a service nova] [instance: da4910c9-6710-436c-8112-1b8030286623] Refreshing network info cache for port 95e236fd-5101-4990-8a09-f52ba45d2a35 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 2059.019431] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199843, 'name': CreateVM_Task, 'duration_secs': 0.313937} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2059.019631] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: da4910c9-6710-436c-8112-1b8030286623] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 2059.020233] env[67752]: DEBUG oslo_concurrency.lockutils [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2059.020403] env[67752]: DEBUG oslo_concurrency.lockutils [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2059.020743] env[67752]: DEBUG oslo_concurrency.lockutils [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 2059.020993] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-48951a94-ad44-4d61-94b0-c7833d3a5798 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2059.025360] env[67752]: DEBUG oslo_vmware.api [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Waiting for the task: (returnval){ [ 2059.025360] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]524a9882-c09b-e217-da6c-885fd4f756c4" [ 2059.025360] env[67752]: _type = "Task" [ 2059.025360] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2059.032773] env[67752]: DEBUG oslo_vmware.api [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]524a9882-c09b-e217-da6c-885fd4f756c4, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2059.290919] env[67752]: DEBUG nova.network.neutron [req-4e2afbd0-00a5-4949-9760-1556dd09882a req-601548a4-9e6f-4387-8654-ea39e7f3229a service nova] [instance: da4910c9-6710-436c-8112-1b8030286623] Updated VIF entry in instance network info cache for port 95e236fd-5101-4990-8a09-f52ba45d2a35. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 2059.291316] env[67752]: DEBUG nova.network.neutron [req-4e2afbd0-00a5-4949-9760-1556dd09882a req-601548a4-9e6f-4387-8654-ea39e7f3229a service nova] [instance: da4910c9-6710-436c-8112-1b8030286623] Updating instance_info_cache with network_info: [{"id": "95e236fd-5101-4990-8a09-f52ba45d2a35", "address": "fa:16:3e:dc:2c:a8", "network": {"id": "bef55da3-1050-4c26-9415-aef048e94b8c", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-327762555-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "f7d3fb6872294f31bd9fa2b48f19808f", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "d6e940e5-e083-4238-973e-f1b4e2a3a5c7", "external-id": "nsx-vlan-transportzone-64", "segmentation_id": 64, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap95e236fd-51", "ovs_interfaceid": "95e236fd-5101-4990-8a09-f52ba45d2a35", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2059.303237] env[67752]: DEBUG oslo_concurrency.lockutils [req-4e2afbd0-00a5-4949-9760-1556dd09882a req-601548a4-9e6f-4387-8654-ea39e7f3229a service nova] Releasing lock "refresh_cache-da4910c9-6710-436c-8112-1b8030286623" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2059.535690] env[67752]: DEBUG oslo_concurrency.lockutils [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2059.535945] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: da4910c9-6710-436c-8112-1b8030286623] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 2059.536120] env[67752]: DEBUG oslo_concurrency.lockutils [None req-883586cb-383a-411e-a6d0-19f839015568 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2063.645598] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2063.657673] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2063.657934] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2063.658163] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2063.658368] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2063.659579] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d8abdab-39eb-45d4-8070-57ee1e13e069 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2063.668940] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2716e22d-40d6-4472-aa59-81e417224094 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2063.684277] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ee82f622-5552-4559-93e4-071f20b9ca6c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2063.691223] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-42be7c1f-ca3b-4187-9cf0-244c0ae8ff4d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2063.721472] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180989MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2063.721638] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2063.721812] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2063.793894] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 34c75d7e-e63e-4eff-afad-0edd61e023fc actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2063.793894] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d52496eb-f8e3-4d86-8454-f56e9097777e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2063.794086] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 09e003b7-3c4c-4b11-a52d-749acf709068 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2063.794138] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fea66a16-92c3-4c7a-8bff-c6eb59270158 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2063.794238] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f2509f7b-1f6b-4942-b591-b7c691fafb26 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2063.794359] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 369dbb1f-3de0-4ba9-af2b-98167a6b4508 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2063.794480] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e623c04f-674f-46b8-9477-400f8421054e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2063.794599] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1a18b0c1-48e6-423d-a43c-6c07ffedbe06 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2063.794716] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 985fe487-26aa-467c-8e65-c0a341fd21ab actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2063.794834] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance da4910c9-6710-436c-8112-1b8030286623 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2063.795052] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 10 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2063.795193] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1792MB phys_disk=200GB used_disk=10GB total_vcpus=48 used_vcpus=10 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2063.926982] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-275794bf-91fa-445d-88c5-5333e4d53b7e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2063.934818] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-06cf6c8b-2078-4592-b3ae-c3ce070e9700 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2063.965277] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2539bc6c-f716-4cad-9c18-32ae0ad143d0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2063.973137] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-710dfeac-5cc1-4d33-b50f-e363b005cbbc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2063.986356] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2063.994243] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2064.007640] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2064.007830] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.286s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2073.635994] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2103.556067] env[67752]: WARNING oslo_vmware.rw_handles [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2103.556067] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2103.556067] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2103.556067] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2103.556067] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2103.556067] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 2103.556067] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2103.556067] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2103.556067] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2103.556067] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2103.556067] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2103.556067] env[67752]: ERROR oslo_vmware.rw_handles [ 2103.556833] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/db61bf1f-f519-4eb4-80d8-df6ce68603fa/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2103.558435] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2103.558691] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Copying Virtual Disk [datastore2] vmware_temp/db61bf1f-f519-4eb4-80d8-df6ce68603fa/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/db61bf1f-f519-4eb4-80d8-df6ce68603fa/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2103.558990] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-45ca7be2-e579-403e-9ca5-12d74b532038 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2103.566346] env[67752]: DEBUG oslo_vmware.api [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Waiting for the task: (returnval){ [ 2103.566346] env[67752]: value = "task-3199844" [ 2103.566346] env[67752]: _type = "Task" [ 2103.566346] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2103.574064] env[67752]: DEBUG oslo_vmware.api [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Task: {'id': task-3199844, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2104.076859] env[67752]: DEBUG oslo_vmware.exceptions [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2104.077150] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2104.077702] env[67752]: ERROR nova.compute.manager [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2104.077702] env[67752]: Faults: ['InvalidArgument'] [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Traceback (most recent call last): [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] yield resources [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] self.driver.spawn(context, instance, image_meta, [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] self._fetch_image_if_missing(context, vi) [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] image_cache(vi, tmp_image_ds_loc) [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] vm_util.copy_virtual_disk( [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] session._wait_for_task(vmdk_copy_task) [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] return self.wait_for_task(task_ref) [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] return evt.wait() [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] result = hub.switch() [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] return self.greenlet.switch() [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] self.f(*self.args, **self.kw) [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] raise exceptions.translate_fault(task_info.error) [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Faults: ['InvalidArgument'] [ 2104.077702] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] [ 2104.078774] env[67752]: INFO nova.compute.manager [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Terminating instance [ 2104.079569] env[67752]: DEBUG oslo_concurrency.lockutils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2104.079779] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2104.080025] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-a2bfb259-b545-4ddf-a44c-91dec4d47665 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2104.082178] env[67752]: DEBUG nova.compute.manager [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 2104.082372] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2104.083090] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0b1cea9f-44b2-4002-a6b7-147ce1ad48c4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2104.089732] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2104.089944] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-e43d3124-4c9d-4647-a8dd-252bb463f4dc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2104.092087] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2104.092266] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2104.093202] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-02375d76-3ff1-4843-b187-2d94e9ebbc62 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2104.097815] env[67752]: DEBUG oslo_vmware.api [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Waiting for the task: (returnval){ [ 2104.097815] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5203c148-5dfb-15e8-29e0-95b334492d80" [ 2104.097815] env[67752]: _type = "Task" [ 2104.097815] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2104.110278] env[67752]: DEBUG oslo_vmware.api [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5203c148-5dfb-15e8-29e0-95b334492d80, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2104.167362] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2104.167592] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2104.167773] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Deleting the datastore file [datastore2] 34c75d7e-e63e-4eff-afad-0edd61e023fc {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2104.168050] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-7b892501-fe0e-476f-967c-deca20772679 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2104.174551] env[67752]: DEBUG oslo_vmware.api [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Waiting for the task: (returnval){ [ 2104.174551] env[67752]: value = "task-3199846" [ 2104.174551] env[67752]: _type = "Task" [ 2104.174551] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2104.181863] env[67752]: DEBUG oslo_vmware.api [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Task: {'id': task-3199846, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2104.607772] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2104.608177] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Creating directory with path [datastore2] vmware_temp/698659fa-cba5-4de1-a7dd-4888cf186b2e/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2104.608283] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-0d80b15e-edf6-428a-bee8-0b7907397017 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2104.619703] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Created directory with path [datastore2] vmware_temp/698659fa-cba5-4de1-a7dd-4888cf186b2e/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2104.619893] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Fetch image to [datastore2] vmware_temp/698659fa-cba5-4de1-a7dd-4888cf186b2e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2104.620080] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/698659fa-cba5-4de1-a7dd-4888cf186b2e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2104.620800] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ddec3d01-35cb-42bf-bec8-c40b7936b7d1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2104.627227] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0056039e-055b-477b-ab12-4789e72167b1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2104.636186] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dac2c164-3edb-4304-9b64-9af54c52a24b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2104.666398] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0fb5eda1-5e19-46a7-b752-411c6940892d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2104.671858] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-185a31fb-1ad3-41ca-8a84-4bc974e49557 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2104.682749] env[67752]: DEBUG oslo_vmware.api [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Task: {'id': task-3199846, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.067669} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2104.683070] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2104.683295] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2104.683515] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2104.683747] env[67752]: INFO nova.compute.manager [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Took 0.60 seconds to destroy the instance on the hypervisor. [ 2104.685979] env[67752]: DEBUG nova.compute.claims [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2104.686159] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2104.686378] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2104.694037] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2104.746374] env[67752]: DEBUG oslo_vmware.rw_handles [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/698659fa-cba5-4de1-a7dd-4888cf186b2e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2104.807447] env[67752]: DEBUG oslo_vmware.rw_handles [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2104.807447] env[67752]: DEBUG oslo_vmware.rw_handles [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/698659fa-cba5-4de1-a7dd-4888cf186b2e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2104.914454] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4ec6d6c-0753-48d1-a38f-ced3c78d385e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2104.921740] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fcef20fd-6654-41a1-911d-5b7332aba6c2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2104.950683] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e12f22c9-2c85-4864-a70c-785f074598d1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2104.957682] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c4e81381-9b88-48a6-80e1-2af5f8042fb5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2104.972396] env[67752]: DEBUG nova.compute.provider_tree [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2104.980813] env[67752]: DEBUG nova.scheduler.client.report [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2104.998850] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.312s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2104.999389] env[67752]: ERROR nova.compute.manager [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2104.999389] env[67752]: Faults: ['InvalidArgument'] [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Traceback (most recent call last): [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] self.driver.spawn(context, instance, image_meta, [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] self._fetch_image_if_missing(context, vi) [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] image_cache(vi, tmp_image_ds_loc) [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] vm_util.copy_virtual_disk( [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] session._wait_for_task(vmdk_copy_task) [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] return self.wait_for_task(task_ref) [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] return evt.wait() [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] result = hub.switch() [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] return self.greenlet.switch() [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] self.f(*self.args, **self.kw) [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] raise exceptions.translate_fault(task_info.error) [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Faults: ['InvalidArgument'] [ 2104.999389] env[67752]: ERROR nova.compute.manager [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] [ 2105.000372] env[67752]: DEBUG nova.compute.utils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2105.001526] env[67752]: DEBUG nova.compute.manager [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Build of instance 34c75d7e-e63e-4eff-afad-0edd61e023fc was re-scheduled: A specified parameter was not correct: fileType [ 2105.001526] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 2105.001892] env[67752]: DEBUG nova.compute.manager [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 2105.006021] env[67752]: DEBUG nova.compute.manager [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 2105.006021] env[67752]: DEBUG nova.compute.manager [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 2105.006021] env[67752]: DEBUG nova.network.neutron [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2105.435752] env[67752]: DEBUG nova.network.neutron [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2105.446026] env[67752]: INFO nova.compute.manager [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Took 0.44 seconds to deallocate network for instance. [ 2105.540238] env[67752]: INFO nova.scheduler.client.report [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Deleted allocations for instance 34c75d7e-e63e-4eff-afad-0edd61e023fc [ 2105.572353] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c452e63a-b462-4713-8c46-57b9d5f93763 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "34c75d7e-e63e-4eff-afad-0edd61e023fc" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 664.239s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2105.573128] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7e62a619-0fef-4f36-87b8-86ef9d6b98fc tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "34c75d7e-e63e-4eff-afad-0edd61e023fc" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 468.899s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2105.573128] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7e62a619-0fef-4f36-87b8-86ef9d6b98fc tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "34c75d7e-e63e-4eff-afad-0edd61e023fc-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2105.573128] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7e62a619-0fef-4f36-87b8-86ef9d6b98fc tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "34c75d7e-e63e-4eff-afad-0edd61e023fc-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2105.573352] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7e62a619-0fef-4f36-87b8-86ef9d6b98fc tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "34c75d7e-e63e-4eff-afad-0edd61e023fc-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2105.575951] env[67752]: INFO nova.compute.manager [None req-7e62a619-0fef-4f36-87b8-86ef9d6b98fc tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Terminating instance [ 2105.578011] env[67752]: DEBUG nova.compute.manager [None req-7e62a619-0fef-4f36-87b8-86ef9d6b98fc tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 2105.578247] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7e62a619-0fef-4f36-87b8-86ef9d6b98fc tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2105.578882] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-ea098d26-439c-4fbd-8253-3156fb0cdfc4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2105.587941] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9698044d-4627-468a-a787-d2a3b35a3098 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2105.616586] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-7e62a619-0fef-4f36-87b8-86ef9d6b98fc tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 34c75d7e-e63e-4eff-afad-0edd61e023fc could not be found. [ 2105.616874] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7e62a619-0fef-4f36-87b8-86ef9d6b98fc tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2105.616954] env[67752]: INFO nova.compute.manager [None req-7e62a619-0fef-4f36-87b8-86ef9d6b98fc tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2105.617214] env[67752]: DEBUG oslo.service.loopingcall [None req-7e62a619-0fef-4f36-87b8-86ef9d6b98fc tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2105.617444] env[67752]: DEBUG nova.compute.manager [-] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 2105.617539] env[67752]: DEBUG nova.network.neutron [-] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2105.639309] env[67752]: DEBUG nova.network.neutron [-] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2105.647322] env[67752]: INFO nova.compute.manager [-] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] Took 0.03 seconds to deallocate network for instance. [ 2105.732554] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7e62a619-0fef-4f36-87b8-86ef9d6b98fc tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "34c75d7e-e63e-4eff-afad-0edd61e023fc" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.160s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2105.734022] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "34c75d7e-e63e-4eff-afad-0edd61e023fc" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 313.824s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2105.734022] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 34c75d7e-e63e-4eff-afad-0edd61e023fc] During sync_power_state the instance has a pending task (deleting). Skip. [ 2105.734206] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "34c75d7e-e63e-4eff-afad-0edd61e023fc" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2109.638618] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2112.635791] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2112.635791] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2114.632831] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2115.634518] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2115.634825] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2116.634152] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2116.634344] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 2119.635395] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2119.635741] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 2119.635741] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 2119.655021] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2119.655021] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2119.655021] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2119.656351] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2119.656351] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2119.656351] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: e623c04f-674f-46b8-9477-400f8421054e] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2119.656351] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2119.656351] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2119.656351] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: da4910c9-6710-436c-8112-1b8030286623] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2119.656351] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 2120.635475] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2124.635618] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2124.649013] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2124.649258] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2124.649432] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2124.649589] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2124.650719] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b19aea87-9592-403a-92a5-a83e69baf47e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2124.659504] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d103965-6bd0-4bfc-8381-ec64a3c9303f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2124.673252] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-bff430cc-9843-42ef-9774-c46b405a9657 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2124.679532] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3b71551d-41f0-4369-b608-43f60ed2c1fb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2124.708620] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180976MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2124.708780] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2124.708985] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2124.773654] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance d52496eb-f8e3-4d86-8454-f56e9097777e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2124.773842] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 09e003b7-3c4c-4b11-a52d-749acf709068 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2124.773980] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fea66a16-92c3-4c7a-8bff-c6eb59270158 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2124.774139] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f2509f7b-1f6b-4942-b591-b7c691fafb26 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2124.774262] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 369dbb1f-3de0-4ba9-af2b-98167a6b4508 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2124.774381] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e623c04f-674f-46b8-9477-400f8421054e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2124.774499] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1a18b0c1-48e6-423d-a43c-6c07ffedbe06 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2124.774615] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 985fe487-26aa-467c-8e65-c0a341fd21ab actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2124.774729] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance da4910c9-6710-436c-8112-1b8030286623 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2124.774914] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 9 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2124.775062] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1664MB phys_disk=200GB used_disk=9GB total_vcpus=48 used_vcpus=9 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2124.874646] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fef7d812-d290-4dbf-a36f-053e1b4b1d2b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2124.883729] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d0178242-fcfb-40ca-9ad7-67a81841a977 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2124.912929] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-20df3a25-9233-474c-84f3-ba0f55b89d9a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2124.919858] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-30c0e4cb-f339-45f8-8c4f-a3826bdb56db {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2124.932556] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2124.940887] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2124.955888] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2124.956102] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.247s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2153.571870] env[67752]: WARNING oslo_vmware.rw_handles [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2153.571870] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2153.571870] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2153.571870] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2153.571870] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2153.571870] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 2153.571870] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2153.571870] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2153.571870] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2153.571870] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2153.571870] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2153.571870] env[67752]: ERROR oslo_vmware.rw_handles [ 2153.572559] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/698659fa-cba5-4de1-a7dd-4888cf186b2e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2153.575371] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2153.575737] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Copying Virtual Disk [datastore2] vmware_temp/698659fa-cba5-4de1-a7dd-4888cf186b2e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/698659fa-cba5-4de1-a7dd-4888cf186b2e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2153.576123] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-180553a0-cb09-4c76-9d27-769b6b3204a8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2153.584058] env[67752]: DEBUG oslo_vmware.api [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Waiting for the task: (returnval){ [ 2153.584058] env[67752]: value = "task-3199847" [ 2153.584058] env[67752]: _type = "Task" [ 2153.584058] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2153.592069] env[67752]: DEBUG oslo_vmware.api [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Task: {'id': task-3199847, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2154.094502] env[67752]: DEBUG oslo_vmware.exceptions [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2154.094732] env[67752]: DEBUG oslo_concurrency.lockutils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2154.095318] env[67752]: ERROR nova.compute.manager [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2154.095318] env[67752]: Faults: ['InvalidArgument'] [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Traceback (most recent call last): [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] yield resources [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self.driver.spawn(context, instance, image_meta, [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self._fetch_image_if_missing(context, vi) [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] image_cache(vi, tmp_image_ds_loc) [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] vm_util.copy_virtual_disk( [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] session._wait_for_task(vmdk_copy_task) [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] return self.wait_for_task(task_ref) [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] return evt.wait() [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] result = hub.switch() [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] return self.greenlet.switch() [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self.f(*self.args, **self.kw) [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] raise exceptions.translate_fault(task_info.error) [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Faults: ['InvalidArgument'] [ 2154.095318] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] [ 2154.096166] env[67752]: INFO nova.compute.manager [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Terminating instance [ 2154.097266] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2154.097480] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2154.097719] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-a4971d24-b12c-46d3-91b6-93957c262caf {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2154.101229] env[67752]: DEBUG nova.compute.manager [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 2154.101421] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2154.102120] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fadc49dd-e3e8-4057-8e85-482752d2034f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2154.108701] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2154.108909] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-719fd419-f4df-4a2a-9139-8252ab0d1534 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2154.110961] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2154.111150] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2154.112077] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-91ce74d4-bbe9-4062-976c-2d4c6577bcff {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2154.116658] env[67752]: DEBUG oslo_vmware.api [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Waiting for the task: (returnval){ [ 2154.116658] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]523f8fc8-a784-49d0-02b7-be44b1ae1d3a" [ 2154.116658] env[67752]: _type = "Task" [ 2154.116658] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2154.123293] env[67752]: DEBUG oslo_vmware.api [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]523f8fc8-a784-49d0-02b7-be44b1ae1d3a, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2154.203722] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2154.203976] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2154.204182] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Deleting the datastore file [datastore2] d52496eb-f8e3-4d86-8454-f56e9097777e {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2154.204445] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-78471769-ceac-414e-80c8-7ae994c33090 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2154.210684] env[67752]: DEBUG oslo_vmware.api [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Waiting for the task: (returnval){ [ 2154.210684] env[67752]: value = "task-3199849" [ 2154.210684] env[67752]: _type = "Task" [ 2154.210684] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2154.218013] env[67752]: DEBUG oslo_vmware.api [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Task: {'id': task-3199849, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2154.627012] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2154.627359] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Creating directory with path [datastore2] vmware_temp/d804ce04-3287-4d61-bad1-b928b8e49e33/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2154.627520] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-0c647709-bfc2-464d-a6b0-f52ed586f17f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2154.638219] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Created directory with path [datastore2] vmware_temp/d804ce04-3287-4d61-bad1-b928b8e49e33/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2154.638413] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Fetch image to [datastore2] vmware_temp/d804ce04-3287-4d61-bad1-b928b8e49e33/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2154.638587] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/d804ce04-3287-4d61-bad1-b928b8e49e33/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2154.639293] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a743c01a-8e57-4ae0-9c08-ab3ac3bf8d62 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2154.645787] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-63f2996f-047a-4d3b-a5cd-171f2dfed26b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2154.654313] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-46f4a5a5-42df-47c4-9d9b-ffa5f1b9c9b5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2154.683811] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-420968f5-1bb9-48d2-b45a-18b63a2e8dd4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2154.688941] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-66332883-9d7c-4b98-98da-45f171f3d296 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2154.710166] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2154.719503] env[67752]: DEBUG oslo_vmware.api [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Task: {'id': task-3199849, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.08806} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2154.719743] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2154.719934] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2154.720114] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2154.720288] env[67752]: INFO nova.compute.manager [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Took 0.62 seconds to destroy the instance on the hypervisor. [ 2154.722354] env[67752]: DEBUG nova.compute.claims [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2154.722529] env[67752]: DEBUG oslo_concurrency.lockutils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2154.722742] env[67752]: DEBUG oslo_concurrency.lockutils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2154.759562] env[67752]: DEBUG oslo_vmware.rw_handles [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/d804ce04-3287-4d61-bad1-b928b8e49e33/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2154.820560] env[67752]: DEBUG oslo_vmware.rw_handles [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2154.820709] env[67752]: DEBUG oslo_vmware.rw_handles [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Closing write handle for https://esx7c1n3.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/d804ce04-3287-4d61-bad1-b928b8e49e33/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2154.936585] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a91e9ef4-4966-4564-8824-0eaf7323dbd0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2154.946171] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c4235ef-42bf-459a-b402-5805f02ff8d2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2154.976667] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c86d921-3293-41cb-9b65-fc3b5455ce7d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2154.983783] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-17c887ac-0303-4055-9fce-dafb58ca6207 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2154.996412] env[67752]: DEBUG nova.compute.provider_tree [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2155.004865] env[67752]: DEBUG nova.scheduler.client.report [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2155.019523] env[67752]: DEBUG oslo_concurrency.lockutils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.297s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2155.020060] env[67752]: ERROR nova.compute.manager [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2155.020060] env[67752]: Faults: ['InvalidArgument'] [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Traceback (most recent call last): [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self.driver.spawn(context, instance, image_meta, [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self._fetch_image_if_missing(context, vi) [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] image_cache(vi, tmp_image_ds_loc) [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] vm_util.copy_virtual_disk( [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] session._wait_for_task(vmdk_copy_task) [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] return self.wait_for_task(task_ref) [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] return evt.wait() [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] result = hub.switch() [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] return self.greenlet.switch() [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self.f(*self.args, **self.kw) [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] raise exceptions.translate_fault(task_info.error) [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Faults: ['InvalidArgument'] [ 2155.020060] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] [ 2155.020774] env[67752]: DEBUG nova.compute.utils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2155.022160] env[67752]: DEBUG nova.compute.manager [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Build of instance d52496eb-f8e3-4d86-8454-f56e9097777e was re-scheduled: A specified parameter was not correct: fileType [ 2155.022160] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 2155.022526] env[67752]: DEBUG nova.compute.manager [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 2155.022698] env[67752]: DEBUG nova.compute.manager [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 2155.022862] env[67752]: DEBUG nova.compute.manager [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 2155.023042] env[67752]: DEBUG nova.network.neutron [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2155.148708] env[67752]: DEBUG neutronclient.v2_0.client [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Error message: {"error": {"code": 401, "title": "Unauthorized", "message": "The request you have made requires authentication."}} {{(pid=67752) _handle_fault_response /opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py:262}} [ 2155.149799] env[67752]: ERROR nova.compute.manager [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Failed to deallocate networks: nova.exception.Unauthorized: Not authorized. [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Traceback (most recent call last): [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self.driver.spawn(context, instance, image_meta, [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self._fetch_image_if_missing(context, vi) [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] image_cache(vi, tmp_image_ds_loc) [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] vm_util.copy_virtual_disk( [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] session._wait_for_task(vmdk_copy_task) [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] return self.wait_for_task(task_ref) [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] return evt.wait() [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] result = hub.switch() [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] return self.greenlet.switch() [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self.f(*self.args, **self.kw) [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] raise exceptions.translate_fault(task_info.error) [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Faults: ['InvalidArgument'] [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] During handling of the above exception, another exception occurred: [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Traceback (most recent call last): [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self._build_and_run_instance(context, instance, image, [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] raise exception.RescheduledException( [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] nova.exception.RescheduledException: Build of instance d52496eb-f8e3-4d86-8454-f56e9097777e was re-scheduled: A specified parameter was not correct: fileType [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Faults: ['InvalidArgument'] [ 2155.149799] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] During handling of the above exception, another exception occurred: [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Traceback (most recent call last): [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] ret = obj(*args, **kwargs) [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] exception_handler_v20(status_code, error_body) [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] raise client_exc(message=error_message, [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Neutron server returns request_ids: ['req-ca79825d-6261-4e43-8f62-457b6ace58ac'] [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] During handling of the above exception, another exception occurred: [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Traceback (most recent call last): [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/compute/manager.py", line 3036, in _cleanup_allocated_networks [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self._deallocate_network(context, instance, requested_networks) [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/compute/manager.py", line 2282, in _deallocate_network [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self.network_api.deallocate_for_instance( [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] data = neutron.list_ports(**search_opts) [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] ret = obj(*args, **kwargs) [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] return self.list('ports', self.ports_path, retrieve_all, [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] ret = obj(*args, **kwargs) [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] for r in self._pagination(collection, path, **params): [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] res = self.get(path, params=params) [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] ret = obj(*args, **kwargs) [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] return self.retry_request("GET", action, body=body, [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] ret = obj(*args, **kwargs) [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 2155.150750] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] return self.do_request(method, action, body=body, [ 2155.151690] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.151690] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] ret = obj(*args, **kwargs) [ 2155.151690] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 2155.151690] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self._handle_fault_response(status_code, replybody, resp) [ 2155.151690] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/network/neutron.py", line 204, in wrapper [ 2155.151690] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] raise exception.Unauthorized() [ 2155.151690] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] nova.exception.Unauthorized: Not authorized. [ 2155.151690] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] [ 2155.200322] env[67752]: INFO nova.scheduler.client.report [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Deleted allocations for instance d52496eb-f8e3-4d86-8454-f56e9097777e [ 2155.222536] env[67752]: DEBUG oslo_concurrency.lockutils [None req-19add632-6864-4ce3-9a8a-3f4f272967b5 tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "d52496eb-f8e3-4d86-8454-f56e9097777e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 534.390s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2155.222814] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "d52496eb-f8e3-4d86-8454-f56e9097777e" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 363.313s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2155.222999] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] During sync_power_state the instance has a pending task (spawning). Skip. [ 2155.223206] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "d52496eb-f8e3-4d86-8454-f56e9097777e" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2155.223442] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7fd00b13-f662-4a6b-a0c6-b0feca8719ba tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "d52496eb-f8e3-4d86-8454-f56e9097777e" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 338.351s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2155.223650] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7fd00b13-f662-4a6b-a0c6-b0feca8719ba tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Acquiring lock "d52496eb-f8e3-4d86-8454-f56e9097777e-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2155.223895] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7fd00b13-f662-4a6b-a0c6-b0feca8719ba tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "d52496eb-f8e3-4d86-8454-f56e9097777e-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2155.224081] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7fd00b13-f662-4a6b-a0c6-b0feca8719ba tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "d52496eb-f8e3-4d86-8454-f56e9097777e-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2155.226043] env[67752]: INFO nova.compute.manager [None req-7fd00b13-f662-4a6b-a0c6-b0feca8719ba tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Terminating instance [ 2155.227710] env[67752]: DEBUG nova.compute.manager [None req-7fd00b13-f662-4a6b-a0c6-b0feca8719ba tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 2155.227916] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7fd00b13-f662-4a6b-a0c6-b0feca8719ba tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2155.228461] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-bf964d1a-32ef-465a-aa14-9d087a98826d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2155.237072] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-eef7f759-50dd-479c-920a-e025fd67a645 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2155.266667] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-7fd00b13-f662-4a6b-a0c6-b0feca8719ba tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance d52496eb-f8e3-4d86-8454-f56e9097777e could not be found. [ 2155.266928] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-7fd00b13-f662-4a6b-a0c6-b0feca8719ba tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2155.267133] env[67752]: INFO nova.compute.manager [None req-7fd00b13-f662-4a6b-a0c6-b0feca8719ba tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2155.267374] env[67752]: DEBUG oslo.service.loopingcall [None req-7fd00b13-f662-4a6b-a0c6-b0feca8719ba tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2155.267586] env[67752]: DEBUG nova.compute.manager [-] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 2155.267682] env[67752]: DEBUG nova.network.neutron [-] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2155.346566] env[67752]: DEBUG neutronclient.v2_0.client [-] Error message: {"error": {"code": 401, "title": "Unauthorized", "message": "The request you have made requires authentication."}} {{(pid=67752) _handle_fault_response /opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py:262}} [ 2155.346856] env[67752]: ERROR nova.network.neutron [-] Neutron client was not able to generate a valid admin token, please verify Neutron admin credential located in nova.conf: neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall [-] Dynamic interval looping call 'oslo_service.loopingcall.RetryDecorator.__call__.._func' failed: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall Traceback (most recent call last): [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall exception_handler_v20(status_code, error_body) [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall raise client_exc(message=error_message, [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall Neutron server returns request_ids: ['req-c8712b58-255a-4d30-9038-ffba3ec66b35'] [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall During handling of the above exception, another exception occurred: [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall Traceback (most recent call last): [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall result = func(*self.args, **self.kw) [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall result = f(*args, **kwargs) [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/compute/manager.py", line 3062, in _deallocate_network_with_retries [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall self._deallocate_network( [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/compute/manager.py", line 2282, in _deallocate_network [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall self.network_api.deallocate_for_instance( [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall data = neutron.list_ports(**search_opts) [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall return self.list('ports', self.ports_path, retrieve_all, [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall for r in self._pagination(collection, path, **params): [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall res = self.get(path, params=params) [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall return self.retry_request("GET", action, body=body, [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall return self.do_request(method, action, body=body, [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall ret = obj(*args, **kwargs) [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall self._handle_fault_response(status_code, replybody, resp) [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall raise exception.NeutronAdminCredentialConfigurationInvalid() [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 2155.347408] env[67752]: ERROR oslo.service.loopingcall [ 2155.348634] env[67752]: ERROR nova.compute.manager [None req-7fd00b13-f662-4a6b-a0c6-b0feca8719ba tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Failed to deallocate network for instance. Error: Networking client is experiencing an unauthorized exception.: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 2155.375980] env[67752]: ERROR nova.compute.manager [None req-7fd00b13-f662-4a6b-a0c6-b0feca8719ba tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Setting instance vm_state to ERROR: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Traceback (most recent call last): [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] ret = obj(*args, **kwargs) [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] exception_handler_v20(status_code, error_body) [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] raise client_exc(message=error_message, [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Neutron server returns request_ids: ['req-c8712b58-255a-4d30-9038-ffba3ec66b35'] [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] During handling of the above exception, another exception occurred: [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Traceback (most recent call last): [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/compute/manager.py", line 3332, in do_terminate_instance [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self._delete_instance(context, instance, bdms) [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/compute/manager.py", line 3267, in _delete_instance [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self._shutdown_instance(context, instance, bdms) [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/compute/manager.py", line 3161, in _shutdown_instance [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self._try_deallocate_network(context, instance, requested_networks) [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/compute/manager.py", line 3075, in _try_deallocate_network [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] with excutils.save_and_reraise_exception(): [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self.force_reraise() [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] raise self.value [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/compute/manager.py", line 3073, in _try_deallocate_network [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] _deallocate_network_with_retries() [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 436, in func [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] return evt.wait() [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] result = hub.switch() [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] return self.greenlet.switch() [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] result = func(*self.args, **self.kw) [ 2155.375980] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] result = f(*args, **kwargs) [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/compute/manager.py", line 3062, in _deallocate_network_with_retries [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self._deallocate_network( [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/compute/manager.py", line 2282, in _deallocate_network [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self.network_api.deallocate_for_instance( [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] data = neutron.list_ports(**search_opts) [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] ret = obj(*args, **kwargs) [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] return self.list('ports', self.ports_path, retrieve_all, [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] ret = obj(*args, **kwargs) [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] for r in self._pagination(collection, path, **params): [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] res = self.get(path, params=params) [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] ret = obj(*args, **kwargs) [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] return self.retry_request("GET", action, body=body, [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] ret = obj(*args, **kwargs) [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] return self.do_request(method, action, body=body, [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] ret = obj(*args, **kwargs) [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] self._handle_fault_response(status_code, replybody, resp) [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] raise exception.NeutronAdminCredentialConfigurationInvalid() [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 2155.376991] env[67752]: ERROR nova.compute.manager [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] [ 2155.402546] env[67752]: DEBUG oslo_concurrency.lockutils [None req-7fd00b13-f662-4a6b-a0c6-b0feca8719ba tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Lock "d52496eb-f8e3-4d86-8454-f56e9097777e" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.179s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2155.445597] env[67752]: INFO nova.compute.manager [None req-7fd00b13-f662-4a6b-a0c6-b0feca8719ba tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] [instance: d52496eb-f8e3-4d86-8454-f56e9097777e] Successfully reverted task state from None on failure for instance. [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server [None req-7fd00b13-f662-4a6b-a0c6-b0feca8719ba tempest-ServersTestMultiNic-523198894 tempest-ServersTestMultiNic-523198894-project-member] Exception during message handling: nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 272, in _handle_fault_response [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server exception_handler_v20(status_code, error_body) [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 90, in exception_handler_v20 [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server raise client_exc(message=error_message, [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server neutronclient.common.exceptions.Unauthorized: 401-{'error': {'code': 401, 'title': 'Unauthorized', 'message': 'The request you have made requires authentication.'}} [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server Neutron server returns request_ids: ['req-c8712b58-255a-4d30-9038-ffba3ec66b35'] [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server During handling of the above exception, another exception occurred: [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server Traceback (most recent call last): [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/server.py", line 165, in _process_incoming [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server res = self.dispatcher.dispatch(message) [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/dispatcher.py", line 309, in dispatch [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server return self._do_dispatch(endpoint, method, ctxt, args) [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_messaging/rpc/dispatcher.py", line 229, in _do_dispatch [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server result = func(ctxt, **new_args) [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/exception_wrapper.py", line 65, in wrapped [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server raise self.value [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/exception_wrapper.py", line 63, in wrapped [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server return f(self, context, *args, **kw) [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 166, in decorated_function [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server raise self.value [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 157, in decorated_function [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/utils.py", line 1453, in decorated_function [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 213, in decorated_function [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server raise self.value [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 203, in decorated_function [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server return function(self, context, *args, **kwargs) [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3344, in terminate_instance [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server do_terminate_instance(instance, bdms) [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py", line 412, in inner [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server return f(*args, **kwargs) [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3339, in do_terminate_instance [ 2155.449313] env[67752]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server raise self.value [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3332, in do_terminate_instance [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server self._delete_instance(context, instance, bdms) [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3267, in _delete_instance [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server self._shutdown_instance(context, instance, bdms) [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3161, in _shutdown_instance [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server self._try_deallocate_network(context, instance, requested_networks) [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3075, in _try_deallocate_network [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server with excutils.save_and_reraise_exception(): [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__ [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server self.force_reraise() [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server raise self.value [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3073, in _try_deallocate_network [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server _deallocate_network_with_retries() [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 436, in func [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server return evt.wait() [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server result = hub.switch() [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server return self.greenlet.switch() [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 150, in _run_loop [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server result = func(*self.args, **self.kw) [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py", line 407, in _func [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server result = f(*args, **kwargs) [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 3062, in _deallocate_network_with_retries [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server self._deallocate_network( [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/compute/manager.py", line 2282, in _deallocate_network [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server self.network_api.deallocate_for_instance( [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 1806, in deallocate_for_instance [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server data = neutron.list_ports(**search_opts) [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 815, in list_ports [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server return self.list('ports', self.ports_path, retrieve_all, [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 372, in list [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server for r in self._pagination(collection, path, **params): [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 387, in _pagination [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server res = self.get(path, params=params) [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 356, in get [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server return self.retry_request("GET", action, body=body, [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 333, in retry_request [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server return self.do_request(method, action, body=body, [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 196, in wrapper [ 2155.450815] env[67752]: ERROR oslo_messaging.rpc.server ret = obj(*args, **kwargs) [ 2155.452289] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/data/venv/lib/python3.10/site-packages/neutronclient/v2_0/client.py", line 297, in do_request [ 2155.452289] env[67752]: ERROR oslo_messaging.rpc.server self._handle_fault_response(status_code, replybody, resp) [ 2155.452289] env[67752]: ERROR oslo_messaging.rpc.server File "/opt/stack/nova/nova/network/neutron.py", line 212, in wrapper [ 2155.452289] env[67752]: ERROR oslo_messaging.rpc.server raise exception.NeutronAdminCredentialConfigurationInvalid() [ 2155.452289] env[67752]: ERROR oslo_messaging.rpc.server nova.exception.NeutronAdminCredentialConfigurationInvalid: Networking client is experiencing an unauthorized exception. [ 2155.452289] env[67752]: ERROR oslo_messaging.rpc.server [ 2169.951247] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2173.636656] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2173.637024] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2175.635312] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2177.636193] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2178.636030] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2178.636293] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 2181.637139] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2181.637139] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 2181.637139] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 2181.678296] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2181.678467] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2181.678604] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2181.678739] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2181.678870] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: e623c04f-674f-46b8-9477-400f8421054e] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2181.678995] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2181.679135] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2181.679262] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: da4910c9-6710-436c-8112-1b8030286623] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2181.679465] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 2181.679974] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2184.635523] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2184.647892] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2184.648133] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2184.648305] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2184.648459] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2184.649875] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f8edde3b-23c3-4c0c-bd95-3ba7bcb2d32c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2184.657752] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cadee4c2-4391-4fca-8924-7972ea7a7691 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2184.671357] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a1e59b1-b934-4a76-a115-c212772d811d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2184.677120] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b242defb-917b-4b88-a91c-7fc2c7b84810 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2184.706485] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181016MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2184.706645] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2184.706830] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2184.769852] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 09e003b7-3c4c-4b11-a52d-749acf709068 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2184.770204] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fea66a16-92c3-4c7a-8bff-c6eb59270158 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2184.770204] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f2509f7b-1f6b-4942-b591-b7c691fafb26 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2184.770287] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 369dbb1f-3de0-4ba9-af2b-98167a6b4508 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2184.770387] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e623c04f-674f-46b8-9477-400f8421054e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2184.770506] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1a18b0c1-48e6-423d-a43c-6c07ffedbe06 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2184.770626] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 985fe487-26aa-467c-8e65-c0a341fd21ab actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2184.770732] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance da4910c9-6710-436c-8112-1b8030286623 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2184.770915] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 8 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2184.771070] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1536MB phys_disk=200GB used_disk=8GB total_vcpus=48 used_vcpus=8 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2184.859864] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c6efaed2-4cfd-429d-98cf-3d7a50a2f069 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2184.867306] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e2651d23-49f8-47bc-aadb-cc50e03a7a2e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2184.896017] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b4095f40-3a88-433f-a5ee-804f33ae5366 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2184.902585] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-f02ca414-ec62-47e8-943f-ab958b82abbf {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2184.915094] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2184.923448] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2184.936547] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2184.936547] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.230s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2194.475039] env[67752]: DEBUG oslo_concurrency.lockutils [None req-69f377ed-1e56-43ff-8f1e-488d7c013ab2 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "1a18b0c1-48e6-423d-a43c-6c07ffedbe06" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2194.538562] env[67752]: DEBUG oslo_concurrency.lockutils [None req-00e0682e-8be9-4b61-81dd-2aa88b5e3edc tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "e623c04f-674f-46b8-9477-400f8421054e" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2203.588868] env[67752]: WARNING oslo_vmware.rw_handles [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2203.588868] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2203.588868] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2203.588868] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2203.588868] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2203.588868] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 2203.588868] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2203.588868] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2203.588868] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2203.588868] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2203.588868] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2203.588868] env[67752]: ERROR oslo_vmware.rw_handles [ 2203.590008] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/d804ce04-3287-4d61-bad1-b928b8e49e33/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2203.592161] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2203.592429] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Copying Virtual Disk [datastore2] vmware_temp/d804ce04-3287-4d61-bad1-b928b8e49e33/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/d804ce04-3287-4d61-bad1-b928b8e49e33/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2203.592718] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-83643463-3870-4115-868a-c003630cc4e5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2203.601375] env[67752]: DEBUG oslo_vmware.api [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Waiting for the task: (returnval){ [ 2203.601375] env[67752]: value = "task-3199850" [ 2203.601375] env[67752]: _type = "Task" [ 2203.601375] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2203.609389] env[67752]: DEBUG oslo_vmware.api [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Task: {'id': task-3199850, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2204.112175] env[67752]: DEBUG oslo_vmware.exceptions [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2204.112449] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2204.113049] env[67752]: ERROR nova.compute.manager [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2204.113049] env[67752]: Faults: ['InvalidArgument'] [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Traceback (most recent call last): [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] yield resources [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] self.driver.spawn(context, instance, image_meta, [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] self._fetch_image_if_missing(context, vi) [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] image_cache(vi, tmp_image_ds_loc) [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] vm_util.copy_virtual_disk( [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] session._wait_for_task(vmdk_copy_task) [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] return self.wait_for_task(task_ref) [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] return evt.wait() [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] result = hub.switch() [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] return self.greenlet.switch() [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] self.f(*self.args, **self.kw) [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] raise exceptions.translate_fault(task_info.error) [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Faults: ['InvalidArgument'] [ 2204.113049] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] [ 2204.113883] env[67752]: INFO nova.compute.manager [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Terminating instance [ 2204.115047] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2204.115162] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2204.115383] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-62be6971-c84a-4dc9-849b-fff9f37fec72 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.117563] env[67752]: DEBUG nova.compute.manager [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 2204.117760] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2204.118473] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-69b3191e-503a-47d7-92ac-fa3b291b0a5c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.124802] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2204.125014] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-0ccc6bb7-ce27-46f8-8386-d045950ad3e4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.127036] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2204.127213] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2204.128149] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-78393cad-2fed-4417-b6c5-9111ca5cfaeb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.132466] env[67752]: DEBUG oslo_vmware.api [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Waiting for the task: (returnval){ [ 2204.132466] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]525c01aa-37dd-f283-9f55-98e129fd3a53" [ 2204.132466] env[67752]: _type = "Task" [ 2204.132466] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2204.139699] env[67752]: DEBUG oslo_vmware.api [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]525c01aa-37dd-f283-9f55-98e129fd3a53, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2204.196472] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2204.196699] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2204.196882] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Deleting the datastore file [datastore2] 09e003b7-3c4c-4b11-a52d-749acf709068 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2204.197159] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-c862eb92-e3a3-426e-a63a-55c75ae2bf61 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.203202] env[67752]: DEBUG oslo_vmware.api [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Waiting for the task: (returnval){ [ 2204.203202] env[67752]: value = "task-3199852" [ 2204.203202] env[67752]: _type = "Task" [ 2204.203202] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2204.210778] env[67752]: DEBUG oslo_vmware.api [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Task: {'id': task-3199852, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2204.642677] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2204.642987] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Creating directory with path [datastore2] vmware_temp/d3418d15-c86d-4fc2-b377-2f5bfd2000ce/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2204.643145] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-b5909ff9-779f-4b83-a5ce-be9cecd4c55c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.653540] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Created directory with path [datastore2] vmware_temp/d3418d15-c86d-4fc2-b377-2f5bfd2000ce/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2204.653749] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Fetch image to [datastore2] vmware_temp/d3418d15-c86d-4fc2-b377-2f5bfd2000ce/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2204.653908] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/d3418d15-c86d-4fc2-b377-2f5bfd2000ce/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2204.654608] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1d2d623-eccd-4dea-b6b5-969f186000ae {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.660664] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7ff2e84e-745a-424c-b05c-9fe1b0e1971a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.670342] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d439fe7b-2f2c-4845-aefa-dc3222c2b453 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.702046] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75de1cdc-54b6-4ab4-9240-bfe86c33665d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.708470] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-b53d6fa8-52c1-47aa-80df-54818556c2a7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.712660] env[67752]: DEBUG oslo_vmware.api [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Task: {'id': task-3199852, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.063835} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2204.713261] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2204.713485] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2204.713683] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2204.713863] env[67752]: INFO nova.compute.manager [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Took 0.60 seconds to destroy the instance on the hypervisor. [ 2204.715944] env[67752]: DEBUG nova.compute.claims [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2204.716118] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2204.716330] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2204.730436] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2204.865651] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4132283c-fc43-48a7-a954-e8cf4ced75aa {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.873125] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ce18b253-c211-4e9c-abd8-7006dcf99f02 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.876661] env[67752]: DEBUG oslo_vmware.rw_handles [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/d3418d15-c86d-4fc2-b377-2f5bfd2000ce/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2204.958426] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5f16a986-c6f2-4961-8fd3-89b1b6296ae1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.961395] env[67752]: DEBUG oslo_vmware.rw_handles [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2204.961571] env[67752]: DEBUG oslo_vmware.rw_handles [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Closing write handle for https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/d3418d15-c86d-4fc2-b377-2f5bfd2000ce/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2204.967926] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c798586f-69bf-4a57-82f7-bb7bc47c60df {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2204.981445] env[67752]: DEBUG nova.compute.provider_tree [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2204.989998] env[67752]: DEBUG nova.scheduler.client.report [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2205.007770] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.291s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2205.008337] env[67752]: ERROR nova.compute.manager [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2205.008337] env[67752]: Faults: ['InvalidArgument'] [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Traceback (most recent call last): [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] self.driver.spawn(context, instance, image_meta, [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] self._fetch_image_if_missing(context, vi) [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] image_cache(vi, tmp_image_ds_loc) [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] vm_util.copy_virtual_disk( [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] session._wait_for_task(vmdk_copy_task) [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] return self.wait_for_task(task_ref) [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] return evt.wait() [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] result = hub.switch() [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] return self.greenlet.switch() [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] self.f(*self.args, **self.kw) [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] raise exceptions.translate_fault(task_info.error) [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Faults: ['InvalidArgument'] [ 2205.008337] env[67752]: ERROR nova.compute.manager [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] [ 2205.009101] env[67752]: DEBUG nova.compute.utils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2205.010437] env[67752]: DEBUG nova.compute.manager [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Build of instance 09e003b7-3c4c-4b11-a52d-749acf709068 was re-scheduled: A specified parameter was not correct: fileType [ 2205.010437] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 2205.010802] env[67752]: DEBUG nova.compute.manager [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 2205.010985] env[67752]: DEBUG nova.compute.manager [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 2205.011171] env[67752]: DEBUG nova.compute.manager [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 2205.011336] env[67752]: DEBUG nova.network.neutron [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2205.275915] env[67752]: DEBUG nova.network.neutron [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2205.289839] env[67752]: INFO nova.compute.manager [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Took 0.28 seconds to deallocate network for instance. [ 2205.382963] env[67752]: INFO nova.scheduler.client.report [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Deleted allocations for instance 09e003b7-3c4c-4b11-a52d-749acf709068 [ 2205.402926] env[67752]: DEBUG oslo_concurrency.lockutils [None req-b0e25a02-240e-496f-a696-c27a40109dca tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "09e003b7-3c4c-4b11-a52d-749acf709068" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 556.363s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2205.403212] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "09e003b7-3c4c-4b11-a52d-749acf709068" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 413.494s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2205.403399] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] During sync_power_state the instance has a pending task (spawning). Skip. [ 2205.403573] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "09e003b7-3c4c-4b11-a52d-749acf709068" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2205.403792] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f1dea674-e4b0-44ba-87de-044ddc21a1c4 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "09e003b7-3c4c-4b11-a52d-749acf709068" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 360.837s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2205.404014] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f1dea674-e4b0-44ba-87de-044ddc21a1c4 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "09e003b7-3c4c-4b11-a52d-749acf709068-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2205.404229] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f1dea674-e4b0-44ba-87de-044ddc21a1c4 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "09e003b7-3c4c-4b11-a52d-749acf709068-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2205.404396] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f1dea674-e4b0-44ba-87de-044ddc21a1c4 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "09e003b7-3c4c-4b11-a52d-749acf709068-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2205.406326] env[67752]: INFO nova.compute.manager [None req-f1dea674-e4b0-44ba-87de-044ddc21a1c4 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Terminating instance [ 2205.408192] env[67752]: DEBUG nova.compute.manager [None req-f1dea674-e4b0-44ba-87de-044ddc21a1c4 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 2205.408434] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f1dea674-e4b0-44ba-87de-044ddc21a1c4 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2205.408952] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-06068eb4-3ad5-4cfd-91c6-325dc5e9e043 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2205.420887] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-01c17b6d-8ed8-499a-b532-c2964e561110 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2205.447054] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-f1dea674-e4b0-44ba-87de-044ddc21a1c4 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 09e003b7-3c4c-4b11-a52d-749acf709068 could not be found. [ 2205.447261] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f1dea674-e4b0-44ba-87de-044ddc21a1c4 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2205.447440] env[67752]: INFO nova.compute.manager [None req-f1dea674-e4b0-44ba-87de-044ddc21a1c4 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2205.447681] env[67752]: DEBUG oslo.service.loopingcall [None req-f1dea674-e4b0-44ba-87de-044ddc21a1c4 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2205.448141] env[67752]: DEBUG nova.compute.manager [-] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 2205.448243] env[67752]: DEBUG nova.network.neutron [-] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2205.469305] env[67752]: DEBUG nova.network.neutron [-] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2205.476951] env[67752]: INFO nova.compute.manager [-] [instance: 09e003b7-3c4c-4b11-a52d-749acf709068] Took 0.03 seconds to deallocate network for instance. [ 2205.557251] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f1dea674-e4b0-44ba-87de-044ddc21a1c4 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "09e003b7-3c4c-4b11-a52d-749acf709068" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.153s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2231.933479] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2234.635668] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2235.635597] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2235.635888] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2238.083300] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9b9d48f5-7463-4763-81f6-db9d1c334208 tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "985fe487-26aa-467c-8e65-c0a341fd21ab" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2238.630855] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2239.634430] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2239.634777] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2239.634777] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 2241.636073] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2241.636073] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 2241.636073] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 2241.660625] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2241.660800] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2241.660946] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2241.661089] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: e623c04f-674f-46b8-9477-400f8421054e] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2241.661324] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2241.661458] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2241.661588] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: da4910c9-6710-436c-8112-1b8030286623] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2241.661709] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 2241.662220] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2246.634712] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2246.647499] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2246.647773] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2246.647999] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2246.648198] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2246.649576] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-56d3316a-6b2d-4419-9ac8-9099fe9e220f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2246.658947] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8f79ed04-2381-42a3-af02-79b7ef9aa9f2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2246.673451] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d48c37a4-1899-462d-a4af-75f33ee41886 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2246.679537] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-662d3178-4e22-4e08-88c7-d34edefa327b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2246.708833] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181001MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2246.708969] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2246.709173] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2246.778159] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance fea66a16-92c3-4c7a-8bff-c6eb59270158 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2246.778326] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance f2509f7b-1f6b-4942-b591-b7c691fafb26 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2246.778454] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 369dbb1f-3de0-4ba9-af2b-98167a6b4508 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2246.778576] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e623c04f-674f-46b8-9477-400f8421054e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2246.778696] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1a18b0c1-48e6-423d-a43c-6c07ffedbe06 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2246.778816] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 985fe487-26aa-467c-8e65-c0a341fd21ab actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2246.778932] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance da4910c9-6710-436c-8112-1b8030286623 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2246.779165] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 7 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2246.779308] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1408MB phys_disk=200GB used_disk=7GB total_vcpus=48 used_vcpus=7 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2246.794880] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Refreshing inventories for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:804}} [ 2246.807778] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Updating ProviderTree inventory for provider c7141b98-bb6c-46bd-b37f-a8e913940199 from _refresh_and_get_inventory using data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) _refresh_and_get_inventory /opt/stack/nova/nova/scheduler/client/report.py:768}} [ 2246.807965] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Updating inventory in ProviderTree for provider c7141b98-bb6c-46bd-b37f-a8e913940199 with inventory: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:176}} [ 2246.818336] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Refreshing aggregate associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, aggregates: None {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:813}} [ 2246.834625] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Refreshing trait associations for resource provider c7141b98-bb6c-46bd-b37f-a8e913940199, traits: COMPUTE_IMAGE_TYPE_VMDK,COMPUTE_SAME_HOST_COLD_MIGRATE,COMPUTE_NET_ATTACH_INTERFACE,COMPUTE_IMAGE_TYPE_ISO,COMPUTE_NODE {{(pid=67752) _refresh_associations /opt/stack/nova/nova/scheduler/client/report.py:825}} [ 2246.916848] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-98a28b1e-4693-4c27-86ee-25acc276ad9d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2246.924208] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-63b7d462-637b-413f-ac0d-fa47cf985cc3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2246.952711] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-dd74c1f4-4a74-47bd-8adc-59d44f0cf2ff {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2246.959414] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5b93226e-13b5-46ce-b3f1-da0cca0b60c1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2246.973151] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2246.983046] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2246.995871] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2246.996071] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.287s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2251.622131] env[67752]: DEBUG oslo_concurrency.lockutils [None req-1603df0b-2bc3-441e-a446-5cd86107bbc4 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Acquiring lock "da4910c9-6710-436c-8112-1b8030286623" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2254.692174] env[67752]: WARNING oslo_vmware.rw_handles [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2254.692174] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2254.692174] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2254.692174] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2254.692174] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2254.692174] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 2254.692174] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2254.692174] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2254.692174] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2254.692174] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2254.692174] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2254.692174] env[67752]: ERROR oslo_vmware.rw_handles [ 2254.692803] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/d3418d15-c86d-4fc2-b377-2f5bfd2000ce/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2254.694972] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2254.695241] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Copying Virtual Disk [datastore2] vmware_temp/d3418d15-c86d-4fc2-b377-2f5bfd2000ce/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/d3418d15-c86d-4fc2-b377-2f5bfd2000ce/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2254.695522] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-492d3290-3bc8-4895-868b-4b478832a24c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2254.703398] env[67752]: DEBUG oslo_vmware.api [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Waiting for the task: (returnval){ [ 2254.703398] env[67752]: value = "task-3199853" [ 2254.703398] env[67752]: _type = "Task" [ 2254.703398] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2254.711373] env[67752]: DEBUG oslo_vmware.api [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Task: {'id': task-3199853, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2255.213797] env[67752]: DEBUG oslo_vmware.exceptions [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2255.214138] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2255.214685] env[67752]: ERROR nova.compute.manager [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2255.214685] env[67752]: Faults: ['InvalidArgument'] [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Traceback (most recent call last): [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] yield resources [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] self.driver.spawn(context, instance, image_meta, [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] self._fetch_image_if_missing(context, vi) [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] image_cache(vi, tmp_image_ds_loc) [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] vm_util.copy_virtual_disk( [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] session._wait_for_task(vmdk_copy_task) [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] return self.wait_for_task(task_ref) [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] return evt.wait() [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] result = hub.switch() [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] return self.greenlet.switch() [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] self.f(*self.args, **self.kw) [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] raise exceptions.translate_fault(task_info.error) [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Faults: ['InvalidArgument'] [ 2255.214685] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] [ 2255.215541] env[67752]: INFO nova.compute.manager [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Terminating instance [ 2255.216616] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2255.216865] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2255.217108] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-3ba3d0ea-5d50-4deb-b6c0-4d64b77bcacb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2255.219349] env[67752]: DEBUG nova.compute.manager [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 2255.219549] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2255.220301] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-90f7b401-a54a-45af-a2c1-aa4f67fa2d9e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2255.227326] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2255.227560] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-3c05031b-f0ee-42eb-872a-fa1f761d5606 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2255.229831] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2255.230013] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2255.230973] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-e359bb6a-16a2-4383-b020-79d6e60f2377 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2255.235634] env[67752]: DEBUG oslo_vmware.api [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Waiting for the task: (returnval){ [ 2255.235634] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]526e883d-05e2-cf43-a3c7-4ae98e2b33a7" [ 2255.235634] env[67752]: _type = "Task" [ 2255.235634] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2255.242675] env[67752]: DEBUG oslo_vmware.api [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]526e883d-05e2-cf43-a3c7-4ae98e2b33a7, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2255.297312] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2255.297530] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2255.297714] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Deleting the datastore file [datastore2] fea66a16-92c3-4c7a-8bff-c6eb59270158 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2255.297975] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-bcea9d4d-4d88-4081-bbe7-09d083a62162 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2255.304377] env[67752]: DEBUG oslo_vmware.api [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Waiting for the task: (returnval){ [ 2255.304377] env[67752]: value = "task-3199855" [ 2255.304377] env[67752]: _type = "Task" [ 2255.304377] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2255.311868] env[67752]: DEBUG oslo_vmware.api [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Task: {'id': task-3199855, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2255.745957] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2255.746332] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Creating directory with path [datastore2] vmware_temp/91533de4-5829-402c-b149-c67e1c0f213a/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2255.746458] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-d7da5b5e-2ee7-4224-b37d-d70bdeb121a4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2255.757892] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Created directory with path [datastore2] vmware_temp/91533de4-5829-402c-b149-c67e1c0f213a/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2255.758134] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Fetch image to [datastore2] vmware_temp/91533de4-5829-402c-b149-c67e1c0f213a/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2255.758328] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/91533de4-5829-402c-b149-c67e1c0f213a/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2255.759054] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d3817275-3cc6-4073-afd3-c6d3e299b107 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2255.766624] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c33a93d8-41ad-46d2-bd9b-d76f2fa40204 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2255.775228] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a955b17d-587c-4951-9d55-1c2613dbaff3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2255.804428] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d850fff-8bb6-4fed-a5c3-5598596b0905 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2255.814594] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-01e0b406-9074-496d-84f5-82f315bb08b9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2255.816244] env[67752]: DEBUG oslo_vmware.api [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Task: {'id': task-3199855, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.076529} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2255.816484] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2255.816688] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2255.816874] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2255.817066] env[67752]: INFO nova.compute.manager [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Took 0.60 seconds to destroy the instance on the hypervisor. [ 2255.819058] env[67752]: DEBUG nova.compute.claims [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2255.819238] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2255.819449] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2255.837418] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2255.957764] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-33089649-b1db-46d3-9bbf-e6248dd4bb3d {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2255.967010] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b6a899c2-f788-4973-9cf4-8e2893d85215 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2256.000390] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f501bfe-cd51-493a-a704-1d742dfec8a5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2256.007273] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d1c4899f-36b1-4d47-950d-26b42c5213a6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2256.020070] env[67752]: DEBUG nova.compute.provider_tree [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2256.021964] env[67752]: DEBUG oslo_vmware.rw_handles [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/91533de4-5829-402c-b149-c67e1c0f213a/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2256.082038] env[67752]: DEBUG nova.scheduler.client.report [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2256.087095] env[67752]: DEBUG oslo_vmware.rw_handles [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2256.087277] env[67752]: DEBUG oslo_vmware.rw_handles [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/91533de4-5829-402c-b149-c67e1c0f213a/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2256.096083] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.276s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2256.096629] env[67752]: ERROR nova.compute.manager [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2256.096629] env[67752]: Faults: ['InvalidArgument'] [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Traceback (most recent call last): [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] self.driver.spawn(context, instance, image_meta, [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] self._fetch_image_if_missing(context, vi) [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] image_cache(vi, tmp_image_ds_loc) [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] vm_util.copy_virtual_disk( [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] session._wait_for_task(vmdk_copy_task) [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] return self.wait_for_task(task_ref) [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] return evt.wait() [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] result = hub.switch() [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] return self.greenlet.switch() [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] self.f(*self.args, **self.kw) [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] raise exceptions.translate_fault(task_info.error) [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Faults: ['InvalidArgument'] [ 2256.096629] env[67752]: ERROR nova.compute.manager [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] [ 2256.097471] env[67752]: DEBUG nova.compute.utils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2256.098866] env[67752]: DEBUG nova.compute.manager [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Build of instance fea66a16-92c3-4c7a-8bff-c6eb59270158 was re-scheduled: A specified parameter was not correct: fileType [ 2256.098866] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 2256.099259] env[67752]: DEBUG nova.compute.manager [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 2256.099430] env[67752]: DEBUG nova.compute.manager [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 2256.099646] env[67752]: DEBUG nova.compute.manager [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 2256.099760] env[67752]: DEBUG nova.network.neutron [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2256.447164] env[67752]: DEBUG nova.network.neutron [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2256.460373] env[67752]: INFO nova.compute.manager [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Took 0.36 seconds to deallocate network for instance. [ 2256.567139] env[67752]: INFO nova.scheduler.client.report [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Deleted allocations for instance fea66a16-92c3-4c7a-8bff-c6eb59270158 [ 2256.589438] env[67752]: DEBUG oslo_concurrency.lockutils [None req-ea5a729c-b5fa-4f0c-af74-4c4c7c7d223e tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Lock "fea66a16-92c3-4c7a-8bff-c6eb59270158" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 598.478s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2256.589749] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "fea66a16-92c3-4c7a-8bff-c6eb59270158" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 464.680s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2256.589961] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] During sync_power_state the instance has a pending task (spawning). Skip. [ 2256.590201] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "fea66a16-92c3-4c7a-8bff-c6eb59270158" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.001s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2256.591050] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f5471ecc-2e83-4753-89bf-69c063096ea9 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Lock "fea66a16-92c3-4c7a-8bff-c6eb59270158" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 403.177s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2256.591156] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f5471ecc-2e83-4753-89bf-69c063096ea9 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Acquiring lock "fea66a16-92c3-4c7a-8bff-c6eb59270158-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2256.591362] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f5471ecc-2e83-4753-89bf-69c063096ea9 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Lock "fea66a16-92c3-4c7a-8bff-c6eb59270158-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2256.591634] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f5471ecc-2e83-4753-89bf-69c063096ea9 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Lock "fea66a16-92c3-4c7a-8bff-c6eb59270158-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2256.593794] env[67752]: INFO nova.compute.manager [None req-f5471ecc-2e83-4753-89bf-69c063096ea9 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Terminating instance [ 2256.595717] env[67752]: DEBUG nova.compute.manager [None req-f5471ecc-2e83-4753-89bf-69c063096ea9 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 2256.595971] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f5471ecc-2e83-4753-89bf-69c063096ea9 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2256.596252] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-4407d0f9-4c08-4726-8eb1-d96e430c816a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2256.607457] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-61378804-82fe-43e0-9f7f-cff4220135de {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2256.633527] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-f5471ecc-2e83-4753-89bf-69c063096ea9 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance fea66a16-92c3-4c7a-8bff-c6eb59270158 could not be found. [ 2256.633734] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f5471ecc-2e83-4753-89bf-69c063096ea9 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2256.633916] env[67752]: INFO nova.compute.manager [None req-f5471ecc-2e83-4753-89bf-69c063096ea9 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2256.634190] env[67752]: DEBUG oslo.service.loopingcall [None req-f5471ecc-2e83-4753-89bf-69c063096ea9 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2256.634659] env[67752]: DEBUG nova.compute.manager [-] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 2256.634760] env[67752]: DEBUG nova.network.neutron [-] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2256.657653] env[67752]: DEBUG nova.network.neutron [-] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2256.665625] env[67752]: INFO nova.compute.manager [-] [instance: fea66a16-92c3-4c7a-8bff-c6eb59270158] Took 0.03 seconds to deallocate network for instance. [ 2256.749149] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f5471ecc-2e83-4753-89bf-69c063096ea9 tempest-AttachInterfacesTestJSON-319601688 tempest-AttachInterfacesTestJSON-319601688-project-member] Lock "fea66a16-92c3-4c7a-8bff-c6eb59270158" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.158s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2291.993925] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2294.635794] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2295.635363] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2297.636790] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2299.635666] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2300.634871] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2300.635143] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 2301.636348] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2301.636743] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 2301.636743] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 2301.660093] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2301.660302] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2301.660368] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: e623c04f-674f-46b8-9477-400f8421054e] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2301.660458] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2301.660583] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2301.660708] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: da4910c9-6710-436c-8112-1b8030286623] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2301.660833] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 2301.661358] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2305.252399] env[67752]: WARNING oslo_vmware.rw_handles [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2305.252399] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2305.252399] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2305.252399] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2305.252399] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2305.252399] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 2305.252399] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2305.252399] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2305.252399] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2305.252399] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2305.252399] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2305.252399] env[67752]: ERROR oslo_vmware.rw_handles [ 2305.253062] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/91533de4-5829-402c-b149-c67e1c0f213a/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2305.254950] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2305.255210] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Copying Virtual Disk [datastore2] vmware_temp/91533de4-5829-402c-b149-c67e1c0f213a/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/91533de4-5829-402c-b149-c67e1c0f213a/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2305.255494] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-b6746ba8-4b6a-4299-b0a3-8538b5b02c07 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2305.263048] env[67752]: DEBUG oslo_vmware.api [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Waiting for the task: (returnval){ [ 2305.263048] env[67752]: value = "task-3199856" [ 2305.263048] env[67752]: _type = "Task" [ 2305.263048] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2305.270573] env[67752]: DEBUG oslo_vmware.api [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Task: {'id': task-3199856, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2305.773130] env[67752]: DEBUG oslo_vmware.exceptions [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2305.773357] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2305.773916] env[67752]: ERROR nova.compute.manager [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2305.773916] env[67752]: Faults: ['InvalidArgument'] [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Traceback (most recent call last): [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] yield resources [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] self.driver.spawn(context, instance, image_meta, [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] self._fetch_image_if_missing(context, vi) [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] image_cache(vi, tmp_image_ds_loc) [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] vm_util.copy_virtual_disk( [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] session._wait_for_task(vmdk_copy_task) [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] return self.wait_for_task(task_ref) [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] return evt.wait() [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] result = hub.switch() [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] return self.greenlet.switch() [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] self.f(*self.args, **self.kw) [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] raise exceptions.translate_fault(task_info.error) [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Faults: ['InvalidArgument'] [ 2305.773916] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] [ 2305.774921] env[67752]: INFO nova.compute.manager [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Terminating instance [ 2305.776436] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2305.776653] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2305.777320] env[67752]: DEBUG nova.compute.manager [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 2305.777513] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2305.777750] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-44e993d7-c484-462c-8f9b-f29a389ce3f2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2305.779978] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-cb449ff2-8a31-4389-9b28-0c46e5ce2e23 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2305.786637] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2305.786842] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-9e187a83-8719-490c-bdeb-f1e8f3323e2e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2305.789023] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2305.789200] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2305.790144] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-fc62bd00-a2b4-4312-995d-ff0c5d80fd43 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2305.794754] env[67752]: DEBUG oslo_vmware.api [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Waiting for the task: (returnval){ [ 2305.794754] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]524e0e1f-feaf-542c-b743-b9831a3244f1" [ 2305.794754] env[67752]: _type = "Task" [ 2305.794754] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2305.801763] env[67752]: DEBUG oslo_vmware.api [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]524e0e1f-feaf-542c-b743-b9831a3244f1, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2305.850439] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2305.850638] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2305.850819] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Deleting the datastore file [datastore2] f2509f7b-1f6b-4942-b591-b7c691fafb26 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2305.851079] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-0f8713e7-87d6-4e77-96f7-bae09bed5d5c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2305.857058] env[67752]: DEBUG oslo_vmware.api [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Waiting for the task: (returnval){ [ 2305.857058] env[67752]: value = "task-3199858" [ 2305.857058] env[67752]: _type = "Task" [ 2305.857058] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2305.864264] env[67752]: DEBUG oslo_vmware.api [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Task: {'id': task-3199858, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2306.305277] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2306.305621] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Creating directory with path [datastore2] vmware_temp/7458e63f-ffd2-4469-8661-971b73e5d143/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2306.305743] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-f511006c-ec41-401f-a62d-b9cf39140944 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2306.316508] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Created directory with path [datastore2] vmware_temp/7458e63f-ffd2-4469-8661-971b73e5d143/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2306.316679] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Fetch image to [datastore2] vmware_temp/7458e63f-ffd2-4469-8661-971b73e5d143/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2306.316865] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/7458e63f-ffd2-4469-8661-971b73e5d143/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2306.317585] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-005439c7-8fcd-4c5d-bcbd-12369b3e7c9b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2306.323641] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3e0f8aca-d65d-47f2-9a71-c251f31b3e56 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2306.332263] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-79b083c3-1937-4725-9b72-022ee19cb1cd {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2306.365477] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9a8824f3-fcc9-4283-ad50-729b59a3d1ac {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2306.371878] env[67752]: DEBUG oslo_vmware.api [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Task: {'id': task-3199858, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.07434} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2306.373219] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2306.373407] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2306.373582] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2306.373750] env[67752]: INFO nova.compute.manager [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Took 0.60 seconds to destroy the instance on the hypervisor. [ 2306.375486] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-7fcf6e8c-83a9-4092-af23-374ee9033c10 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2306.377328] env[67752]: DEBUG nova.compute.claims [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2306.377504] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2306.377732] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2306.400712] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2306.503702] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d844142-db66-4aaa-b8fa-b3310bb84366 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2306.514084] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-44056fd9-988c-4a82-a037-55b83bbf1289 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2306.544858] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-d0f20612-cba2-428f-9686-8e70be59a82c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2306.551895] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e60f4857-e688-4064-aa32-e37839d71cd3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2306.556913] env[67752]: DEBUG oslo_vmware.rw_handles [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/7458e63f-ffd2-4469-8661-971b73e5d143/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2306.565761] env[67752]: DEBUG nova.compute.provider_tree [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2306.623856] env[67752]: DEBUG nova.scheduler.client.report [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2306.628772] env[67752]: DEBUG oslo_vmware.rw_handles [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2306.628965] env[67752]: DEBUG oslo_vmware.rw_handles [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Closing write handle for https://esx7c2n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/7458e63f-ffd2-4469-8661-971b73e5d143/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2306.639087] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.261s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2306.639557] env[67752]: ERROR nova.compute.manager [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2306.639557] env[67752]: Faults: ['InvalidArgument'] [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Traceback (most recent call last): [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] self.driver.spawn(context, instance, image_meta, [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] self._fetch_image_if_missing(context, vi) [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] image_cache(vi, tmp_image_ds_loc) [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] vm_util.copy_virtual_disk( [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] session._wait_for_task(vmdk_copy_task) [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] return self.wait_for_task(task_ref) [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] return evt.wait() [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] result = hub.switch() [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] return self.greenlet.switch() [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] self.f(*self.args, **self.kw) [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] raise exceptions.translate_fault(task_info.error) [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Faults: ['InvalidArgument'] [ 2306.639557] env[67752]: ERROR nova.compute.manager [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] [ 2306.640415] env[67752]: DEBUG nova.compute.utils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2306.642736] env[67752]: DEBUG nova.compute.manager [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Build of instance f2509f7b-1f6b-4942-b591-b7c691fafb26 was re-scheduled: A specified parameter was not correct: fileType [ 2306.642736] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 2306.643159] env[67752]: DEBUG nova.compute.manager [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 2306.643354] env[67752]: DEBUG nova.compute.manager [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 2306.643545] env[67752]: DEBUG nova.compute.manager [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 2306.643767] env[67752]: DEBUG nova.network.neutron [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2307.087734] env[67752]: DEBUG nova.network.neutron [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2307.098898] env[67752]: INFO nova.compute.manager [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Took 0.45 seconds to deallocate network for instance. [ 2307.198465] env[67752]: INFO nova.scheduler.client.report [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Deleted allocations for instance f2509f7b-1f6b-4942-b591-b7c691fafb26 [ 2307.219252] env[67752]: DEBUG oslo_concurrency.lockutils [None req-fe779f7f-e9f7-43c4-b271-cd7710231502 tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "f2509f7b-1f6b-4942-b591-b7c691fafb26" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 570.948s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2307.219520] env[67752]: DEBUG oslo_concurrency.lockutils [None req-df260266-bafa-4aec-b55b-68047119aacf tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "f2509f7b-1f6b-4942-b591-b7c691fafb26" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 375.135s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2307.219742] env[67752]: DEBUG oslo_concurrency.lockutils [None req-df260266-bafa-4aec-b55b-68047119aacf tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Acquiring lock "f2509f7b-1f6b-4942-b591-b7c691fafb26-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2307.219954] env[67752]: DEBUG oslo_concurrency.lockutils [None req-df260266-bafa-4aec-b55b-68047119aacf tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "f2509f7b-1f6b-4942-b591-b7c691fafb26-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2307.220150] env[67752]: DEBUG oslo_concurrency.lockutils [None req-df260266-bafa-4aec-b55b-68047119aacf tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "f2509f7b-1f6b-4942-b591-b7c691fafb26-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2307.222447] env[67752]: INFO nova.compute.manager [None req-df260266-bafa-4aec-b55b-68047119aacf tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Terminating instance [ 2307.224769] env[67752]: DEBUG nova.compute.manager [None req-df260266-bafa-4aec-b55b-68047119aacf tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 2307.224973] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-df260266-bafa-4aec-b55b-68047119aacf tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2307.225257] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-8a7c4885-be40-4c04-9bc2-0f6773a73d17 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2307.234222] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-95fb8a4a-9d71-4d69-bf2f-d356001f772b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2307.260670] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-df260266-bafa-4aec-b55b-68047119aacf tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance f2509f7b-1f6b-4942-b591-b7c691fafb26 could not be found. [ 2307.260875] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-df260266-bafa-4aec-b55b-68047119aacf tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2307.261077] env[67752]: INFO nova.compute.manager [None req-df260266-bafa-4aec-b55b-68047119aacf tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2307.261325] env[67752]: DEBUG oslo.service.loopingcall [None req-df260266-bafa-4aec-b55b-68047119aacf tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2307.261834] env[67752]: DEBUG nova.compute.manager [-] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 2307.261955] env[67752]: DEBUG nova.network.neutron [-] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2307.287050] env[67752]: DEBUG nova.network.neutron [-] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2307.295414] env[67752]: INFO nova.compute.manager [-] [instance: f2509f7b-1f6b-4942-b591-b7c691fafb26] Took 0.03 seconds to deallocate network for instance. [ 2307.384557] env[67752]: DEBUG oslo_concurrency.lockutils [None req-df260266-bafa-4aec-b55b-68047119aacf tempest-AttachVolumeTestJSON-141172378 tempest-AttachVolumeTestJSON-141172378-project-member] Lock "f2509f7b-1f6b-4942-b591-b7c691fafb26" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.165s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2308.635073] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2308.646367] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2308.646636] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2308.646753] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2308.646935] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2308.648055] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-af42249f-2eb5-4194-a863-e88a3881646a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2308.656754] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-71e0010d-bd3e-4b84-a202-8aa8cda28c71 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2308.670157] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-67980b86-9182-4309-b6bf-70993faf498f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2308.676143] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c23b6295-9f55-4db2-bd56-61c5470e597a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2308.703953] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=181002MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2308.704128] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2308.704315] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2308.758428] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 369dbb1f-3de0-4ba9-af2b-98167a6b4508 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2308.758428] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e623c04f-674f-46b8-9477-400f8421054e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2308.758428] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1a18b0c1-48e6-423d-a43c-6c07ffedbe06 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2308.758428] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 985fe487-26aa-467c-8e65-c0a341fd21ab actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2308.758428] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance da4910c9-6710-436c-8112-1b8030286623 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2308.758428] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 5 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2308.758428] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1152MB phys_disk=200GB used_disk=5GB total_vcpus=48 used_vcpus=5 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2308.821967] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-425694d6-762e-46d9-a1f9-e85db9e539d6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2308.829465] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-991a6e60-3b63-472e-be30-0e702c8d50b0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2308.859176] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4983f584-a389-49a8-ae67-281bdabc2159 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2308.865881] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5cb24033-dd0b-4c52-a78b-608aca165230 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2308.878671] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2308.887060] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2308.900659] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2308.900835] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.197s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2353.897450] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2354.074597] env[67752]: WARNING oslo_vmware.rw_handles [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2354.074597] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2354.074597] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2354.074597] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2354.074597] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2354.074597] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 2354.074597] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2354.074597] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2354.074597] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2354.074597] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2354.074597] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2354.074597] env[67752]: ERROR oslo_vmware.rw_handles [ 2354.075112] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/7458e63f-ffd2-4469-8661-971b73e5d143/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2354.077230] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2354.077504] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Copying Virtual Disk [datastore2] vmware_temp/7458e63f-ffd2-4469-8661-971b73e5d143/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/7458e63f-ffd2-4469-8661-971b73e5d143/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2354.077833] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-f84cea60-9828-4343-8a56-dbc2754d2332 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2354.085395] env[67752]: DEBUG oslo_vmware.api [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Waiting for the task: (returnval){ [ 2354.085395] env[67752]: value = "task-3199859" [ 2354.085395] env[67752]: _type = "Task" [ 2354.085395] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2354.093542] env[67752]: DEBUG oslo_vmware.api [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Task: {'id': task-3199859, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2354.596141] env[67752]: DEBUG oslo_vmware.exceptions [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2354.596435] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2354.597071] env[67752]: ERROR nova.compute.manager [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2354.597071] env[67752]: Faults: ['InvalidArgument'] [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Traceback (most recent call last): [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] yield resources [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] self.driver.spawn(context, instance, image_meta, [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] self._fetch_image_if_missing(context, vi) [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] image_cache(vi, tmp_image_ds_loc) [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] vm_util.copy_virtual_disk( [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] session._wait_for_task(vmdk_copy_task) [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] return self.wait_for_task(task_ref) [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] return evt.wait() [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] result = hub.switch() [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] return self.greenlet.switch() [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] self.f(*self.args, **self.kw) [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] raise exceptions.translate_fault(task_info.error) [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Faults: ['InvalidArgument'] [ 2354.597071] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] [ 2354.597902] env[67752]: INFO nova.compute.manager [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Terminating instance [ 2354.598993] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2354.599217] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2354.599450] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-b7cccaec-f3df-4253-9ba6-071e4d139358 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2354.601615] env[67752]: DEBUG nova.compute.manager [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 2354.601810] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2354.602537] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-29a33e9f-b861-4651-971c-1d2f58603b31 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2354.608964] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2354.609199] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-dfd0442e-1dfa-4a5e-9de8-fbf6837e0313 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2354.611304] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2354.611481] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2354.612422] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-e84f3b40-0f0a-410f-9cb4-092b6d1a518a {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2354.617064] env[67752]: DEBUG oslo_vmware.api [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for the task: (returnval){ [ 2354.617064] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52e7fbcd-d631-dd9b-9c72-930d11dcecc0" [ 2354.617064] env[67752]: _type = "Task" [ 2354.617064] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2354.625113] env[67752]: DEBUG oslo_vmware.api [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]52e7fbcd-d631-dd9b-9c72-930d11dcecc0, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2354.774449] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2354.774674] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2354.774835] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Deleting the datastore file [datastore2] 369dbb1f-3de0-4ba9-af2b-98167a6b4508 {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2354.775137] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-1a8de692-43f9-40fd-a7dd-1e834d425ac0 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2354.780745] env[67752]: DEBUG oslo_vmware.api [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Waiting for the task: (returnval){ [ 2354.780745] env[67752]: value = "task-3199861" [ 2354.780745] env[67752]: _type = "Task" [ 2354.780745] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2354.788080] env[67752]: DEBUG oslo_vmware.api [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Task: {'id': task-3199861, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2355.126504] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2355.126828] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Creating directory with path [datastore2] vmware_temp/ac219cd8-e200-4a43-95da-aad4627a822e/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2355.126965] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-22310332-90d7-44e7-bc35-3bfdd0e78ce8 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2355.137440] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Created directory with path [datastore2] vmware_temp/ac219cd8-e200-4a43-95da-aad4627a822e/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2355.137624] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Fetch image to [datastore2] vmware_temp/ac219cd8-e200-4a43-95da-aad4627a822e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2355.137795] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/ac219cd8-e200-4a43-95da-aad4627a822e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2355.138521] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9ba9e00b-8ab3-4e84-b673-8a3997557278 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2355.144663] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-17423098-a8b9-4be7-8e63-e5fca3d78028 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2355.153634] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-094fbf7d-ac9a-4163-bb9c-5063a88a4e36 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2355.182909] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-aa41e0a0-4c75-400a-89a4-c403bf01447e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2355.188066] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-b6fd91c9-a8f9-49e4-8ee2-d032bcf6e7f4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2355.209013] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2355.257705] env[67752]: DEBUG oslo_vmware.rw_handles [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/ac219cd8-e200-4a43-95da-aad4627a822e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2355.318413] env[67752]: DEBUG oslo_vmware.rw_handles [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2355.318588] env[67752]: DEBUG oslo_vmware.rw_handles [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Closing write handle for https://esx7c1n2.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/ac219cd8-e200-4a43-95da-aad4627a822e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2355.321951] env[67752]: DEBUG oslo_vmware.api [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Task: {'id': task-3199861, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.082086} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2355.322243] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2355.322448] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2355.322652] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2355.322840] env[67752]: INFO nova.compute.manager [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Took 0.72 seconds to destroy the instance on the hypervisor. [ 2355.325062] env[67752]: DEBUG nova.compute.claims [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2355.325244] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2355.325477] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2355.515955] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-93df5f44-683b-46c9-a6b2-ab546bc43ab2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2355.523107] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-09578ca9-cb2a-4abb-8e04-ecf825e7d7c9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2355.553013] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1d353309-9dfc-4a00-809b-f0e65c62deeb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2355.559471] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4e5bfc93-665c-4642-b026-797f74eb5baa {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2355.571911] env[67752]: DEBUG nova.compute.provider_tree [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2355.580423] env[67752]: DEBUG nova.scheduler.client.report [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2355.595439] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.270s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2355.595997] env[67752]: ERROR nova.compute.manager [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2355.595997] env[67752]: Faults: ['InvalidArgument'] [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Traceback (most recent call last): [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] self.driver.spawn(context, instance, image_meta, [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] self._fetch_image_if_missing(context, vi) [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] image_cache(vi, tmp_image_ds_loc) [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] vm_util.copy_virtual_disk( [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] session._wait_for_task(vmdk_copy_task) [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] return self.wait_for_task(task_ref) [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] return evt.wait() [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] result = hub.switch() [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] return self.greenlet.switch() [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] self.f(*self.args, **self.kw) [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] raise exceptions.translate_fault(task_info.error) [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Faults: ['InvalidArgument'] [ 2355.595997] env[67752]: ERROR nova.compute.manager [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] [ 2355.597113] env[67752]: DEBUG nova.compute.utils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2355.598502] env[67752]: DEBUG nova.compute.manager [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Build of instance 369dbb1f-3de0-4ba9-af2b-98167a6b4508 was re-scheduled: A specified parameter was not correct: fileType [ 2355.598502] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 2355.598886] env[67752]: DEBUG nova.compute.manager [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 2355.599072] env[67752]: DEBUG nova.compute.manager [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 2355.599245] env[67752]: DEBUG nova.compute.manager [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 2355.599411] env[67752]: DEBUG nova.network.neutron [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2355.634199] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2356.183578] env[67752]: DEBUG nova.network.neutron [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2356.193807] env[67752]: INFO nova.compute.manager [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Took 0.59 seconds to deallocate network for instance. [ 2356.297715] env[67752]: INFO nova.scheduler.client.report [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Deleted allocations for instance 369dbb1f-3de0-4ba9-af2b-98167a6b4508 [ 2356.318044] env[67752]: DEBUG oslo_concurrency.lockutils [None req-3867c345-dcff-4a05-9bf9-954bc7969df5 tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "369dbb1f-3de0-4ba9-af2b-98167a6b4508" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 522.133s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2356.318282] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f8c5d39b-786a-4b96-8ece-4d4a8e6e493b tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "369dbb1f-3de0-4ba9-af2b-98167a6b4508" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 325.268s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2356.318474] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f8c5d39b-786a-4b96-8ece-4d4a8e6e493b tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Acquiring lock "369dbb1f-3de0-4ba9-af2b-98167a6b4508-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2356.318722] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f8c5d39b-786a-4b96-8ece-4d4a8e6e493b tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "369dbb1f-3de0-4ba9-af2b-98167a6b4508-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2356.318931] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f8c5d39b-786a-4b96-8ece-4d4a8e6e493b tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "369dbb1f-3de0-4ba9-af2b-98167a6b4508-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2356.321785] env[67752]: INFO nova.compute.manager [None req-f8c5d39b-786a-4b96-8ece-4d4a8e6e493b tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Terminating instance [ 2356.323629] env[67752]: DEBUG nova.compute.manager [None req-f8c5d39b-786a-4b96-8ece-4d4a8e6e493b tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 2356.323869] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f8c5d39b-786a-4b96-8ece-4d4a8e6e493b tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2356.324394] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-b5be4100-41c7-433a-bbca-e28856777ba9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2356.333250] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-6d844d85-a9bf-47d4-a0ed-e0ce702ea0f4 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2356.359652] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-f8c5d39b-786a-4b96-8ece-4d4a8e6e493b tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance 369dbb1f-3de0-4ba9-af2b-98167a6b4508 could not be found. [ 2356.359861] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f8c5d39b-786a-4b96-8ece-4d4a8e6e493b tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2356.360060] env[67752]: INFO nova.compute.manager [None req-f8c5d39b-786a-4b96-8ece-4d4a8e6e493b tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2356.360307] env[67752]: DEBUG oslo.service.loopingcall [None req-f8c5d39b-786a-4b96-8ece-4d4a8e6e493b tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2356.360744] env[67752]: DEBUG nova.compute.manager [-] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 2356.360841] env[67752]: DEBUG nova.network.neutron [-] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2356.384014] env[67752]: DEBUG nova.network.neutron [-] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2356.391939] env[67752]: INFO nova.compute.manager [-] [instance: 369dbb1f-3de0-4ba9-af2b-98167a6b4508] Took 0.03 seconds to deallocate network for instance. [ 2356.472230] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f8c5d39b-786a-4b96-8ece-4d4a8e6e493b tempest-AttachVolumeNegativeTest-101442029 tempest-AttachVolumeNegativeTest-101442029-project-member] Lock "369dbb1f-3de0-4ba9-af2b-98167a6b4508" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.154s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2356.634922] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2357.635307] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2358.635062] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._run_pending_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2358.635062] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Cleaning up deleted instances {{(pid=67752) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11219}} [ 2358.645588] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] There are 0 instances to clean {{(pid=67752) _run_pending_deletes /opt/stack/nova/nova/compute/manager.py:11228}} [ 2359.635261] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2359.635533] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._cleanup_incomplete_migrations {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2359.635680] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Cleaning up deleted instances with incomplete migration {{(pid=67752) _cleanup_incomplete_migrations /opt/stack/nova/nova/compute/manager.py:11257}} [ 2361.638764] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._sync_scheduler_instance_info {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2361.656748] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2361.656936] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 2361.656936] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 2361.670446] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: e623c04f-674f-46b8-9477-400f8421054e] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2361.670616] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2361.670748] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2361.670877] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: da4910c9-6710-436c-8112-1b8030286623] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2361.671013] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 2361.671523] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2361.671697] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 2363.635386] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2369.634354] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2369.646820] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2369.647056] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2369.647243] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2369.647405] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2369.648890] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-28fa4b76-7a35-465f-b40c-6bb29856a86e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2369.657228] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d67c98b-cbfc-4a69-a6b5-f21dbf48eda9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2369.673047] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-8c17d677-b102-4775-94f9-c282fcf90627 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2369.679876] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-c19f1bf3-3c7f-4e71-a2ca-aa5b80667a88 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2369.709260] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180980MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2369.709424] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2369.709635] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2369.774169] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance e623c04f-674f-46b8-9477-400f8421054e actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2369.774338] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1a18b0c1-48e6-423d-a43c-6c07ffedbe06 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2369.774473] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 985fe487-26aa-467c-8e65-c0a341fd21ab actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2369.774597] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance da4910c9-6710-436c-8112-1b8030286623 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2369.774786] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 4 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2369.774927] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=1024MB phys_disk=200GB used_disk=4GB total_vcpus=48 used_vcpus=4 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2369.841899] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a6451576-ea00-4d4e-969b-f6421c92bb8f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2369.849329] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-de0add59-4a6d-485f-8e28-81194c5145e3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2369.878878] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e5f1de31-5305-4f63-8d33-719e314847f9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2369.885931] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0be64a9d-bf10-4c67-b9f5-5d0ddbd9ac22 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2369.898565] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2369.907613] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2369.924632] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2369.924819] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.215s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2378.891024] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._cleanup_running_deleted_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2378.891359] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Getting list of instances from cluster (obj){ [ 2378.891359] env[67752]: value = "domain-c8" [ 2378.891359] env[67752]: _type = "ClusterComputeResource" [ 2378.891359] env[67752]: } {{(pid=67752) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 2378.892485] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-820e9762-5009-4d3b-80b0-4764d57af065 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2378.904845] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Got total of 4 instances {{(pid=67752) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 2387.635651] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._cleanup_expired_console_auth_tokens {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2404.746982] env[67752]: WARNING oslo_vmware.rw_handles [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Error occurred while reading the HTTP response.: http.client.RemoteDisconnected: Remote end closed connection without response [ 2404.746982] env[67752]: ERROR oslo_vmware.rw_handles Traceback (most recent call last): [ 2404.746982] env[67752]: ERROR oslo_vmware.rw_handles File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py", line 283, in close [ 2404.746982] env[67752]: ERROR oslo_vmware.rw_handles self._conn.getresponse() [ 2404.746982] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 1375, in getresponse [ 2404.746982] env[67752]: ERROR oslo_vmware.rw_handles response.begin() [ 2404.746982] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 318, in begin [ 2404.746982] env[67752]: ERROR oslo_vmware.rw_handles version, status, reason = self._read_status() [ 2404.746982] env[67752]: ERROR oslo_vmware.rw_handles File "/usr/lib/python3.10/http/client.py", line 287, in _read_status [ 2404.746982] env[67752]: ERROR oslo_vmware.rw_handles raise RemoteDisconnected("Remote end closed connection without" [ 2404.746982] env[67752]: ERROR oslo_vmware.rw_handles http.client.RemoteDisconnected: Remote end closed connection without response [ 2404.746982] env[67752]: ERROR oslo_vmware.rw_handles [ 2404.747834] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Downloaded image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to vmware_temp/ac219cd8-e200-4a43-95da-aad4627a822e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:258}} [ 2404.749305] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Caching image {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:638}} [ 2404.749557] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Copying Virtual Disk [datastore2] vmware_temp/ac219cd8-e200-4a43-95da-aad4627a822e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk to [datastore2] vmware_temp/ac219cd8-e200-4a43-95da-aad4627a822e/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk {{(pid=67752) copy_virtual_disk /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1413}} [ 2404.749849] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualDiskManager.CopyVirtualDisk_Task with opID=oslo.vmware-51da302d-bcaa-4f73-b38c-702634d9e468 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2404.757943] env[67752]: DEBUG oslo_vmware.api [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for the task: (returnval){ [ 2404.757943] env[67752]: value = "task-3199862" [ 2404.757943] env[67752]: _type = "Task" [ 2404.757943] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2404.765758] env[67752]: DEBUG oslo_vmware.api [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Task: {'id': task-3199862, 'name': CopyVirtualDisk_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2405.268849] env[67752]: DEBUG oslo_vmware.exceptions [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Fault InvalidArgument not matched. {{(pid=67752) get_fault_class /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/exceptions.py:290}} [ 2405.269152] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2405.269710] env[67752]: ERROR nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Instance failed to spawn: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2405.269710] env[67752]: Faults: ['InvalidArgument'] [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] Traceback (most recent call last): [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/nova/nova/compute/manager.py", line 2885, in _build_resources [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] yield resources [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] self.driver.spawn(context, instance, image_meta, [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] self._fetch_image_if_missing(context, vi) [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] image_cache(vi, tmp_image_ds_loc) [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] vm_util.copy_virtual_disk( [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] session._wait_for_task(vmdk_copy_task) [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] return self.wait_for_task(task_ref) [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] return evt.wait() [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] result = hub.switch() [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] return self.greenlet.switch() [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] self.f(*self.args, **self.kw) [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] raise exceptions.translate_fault(task_info.error) [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] Faults: ['InvalidArgument'] [ 2405.269710] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] [ 2405.270789] env[67752]: INFO nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Terminating instance [ 2405.271602] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2405.271831] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Creating directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2405.272083] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-a435742b-f88b-4110-960a-8fc3d9ab10cb {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2405.274237] env[67752]: DEBUG nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 2405.274463] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2405.275210] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-47f93ae7-5d1c-45e8-b6da-240ef7535db1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2405.282015] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Unregistering the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1094}} [ 2405.282242] env[67752]: DEBUG oslo_vmware.service [-] Invoking VirtualMachine.UnregisterVM with opID=oslo.vmware-ed50d86d-5796-4a8f-9a3d-9183805af608 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2405.284364] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Created directory with path [datastore2] devstack-image-cache_base {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2405.284566] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Folder [datastore2] devstack-image-cache_base created. {{(pid=67752) _create_folder_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1851}} [ 2405.285492] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-49121fc3-44b8-4a12-9aba-163ee23919e3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2405.290659] env[67752]: DEBUG oslo_vmware.api [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for the task: (returnval){ [ 2405.290659] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]524917b9-a7d1-916f-a005-b0883e835522" [ 2405.290659] env[67752]: _type = "Task" [ 2405.290659] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2405.297823] env[67752]: DEBUG oslo_vmware.api [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]524917b9-a7d1-916f-a005-b0883e835522, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2405.346582] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Unregistered the VM {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1097}} [ 2405.346827] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Deleting contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1107}} [ 2405.347073] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Deleting the datastore file [datastore2] e623c04f-674f-46b8-9477-400f8421054e {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:211}} [ 2405.347301] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.DeleteDatastoreFile_Task with opID=oslo.vmware-065d6c59-2039-4b31-80ed-fd5faabe206c {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2405.353638] env[67752]: DEBUG oslo_vmware.api [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for the task: (returnval){ [ 2405.353638] env[67752]: value = "task-3199864" [ 2405.353638] env[67752]: _type = "Task" [ 2405.353638] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2405.361018] env[67752]: DEBUG oslo_vmware.api [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Task: {'id': task-3199864, 'name': DeleteDatastoreFile_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2405.801352] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Preparing fetch location {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:633}} [ 2405.801700] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Creating directory with path [datastore2] vmware_temp/f47866f3-a8c6-4abf-8e33-fb702db155d8/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:399}} [ 2405.801806] env[67752]: DEBUG oslo_vmware.service [-] Invoking FileManager.MakeDirectory with opID=oslo.vmware-de0861ef-cc4b-4675-8f9c-c72980bebbfc {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2405.812690] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Created directory with path [datastore2] vmware_temp/f47866f3-a8c6-4abf-8e33-fb702db155d8/5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) mkdir /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:404}} [ 2405.812880] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Fetch image to [datastore2] vmware_temp/f47866f3-a8c6-4abf-8e33-fb702db155d8/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:635}} [ 2405.813066] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to [datastore2] vmware_temp/f47866f3-a8c6-4abf-8e33-fb702db155d8/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk on the data store datastore2 {{(pid=67752) _fetch_image_as_file /opt/stack/nova/nova/virt/vmwareapi/vmops.py:399}} [ 2405.813765] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5cf18bce-9a6e-4df1-bac4-233b915a28f2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2405.819886] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-75333a25-37d0-4771-a70c-19c38db2ff69 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2405.829444] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-3ce4be5c-c5d3-4ea9-a535-3073df99e6d5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2405.861790] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-16ba6e8e-338a-48c5-a456-31a37ffbd78e {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2405.864195] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._sync_power_states {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2405.869948] env[67752]: DEBUG oslo_vmware.api [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Task: {'id': task-3199864, 'name': DeleteDatastoreFile_Task, 'duration_secs': 0.078093} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2405.871404] env[67752]: DEBUG nova.virt.vmwareapi.ds_util [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Deleted the datastore file {{(pid=67752) file_delete /opt/stack/nova/nova/virt/vmwareapi/ds_util.py:220}} [ 2405.871603] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Deleted contents of the VM from datastore datastore2 {{(pid=67752) _destroy_instance /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1117}} [ 2405.871776] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2405.871948] env[67752]: INFO nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Took 0.60 seconds to destroy the instance on the hypervisor. [ 2405.873636] env[67752]: DEBUG oslo_vmware.service [-] Invoking SessionManager.AcquireGenericServiceTicket with opID=oslo.vmware-7bdffd5d-7d8e-48e6-9361-2baedc557cb3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2405.875496] env[67752]: DEBUG nova.compute.claims [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Aborting claim: {{(pid=67752) abort /opt/stack/nova/nova/compute/claims.py:85}} [ 2405.875671] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2405.875885] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2405.880700] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Getting list of instances from cluster (obj){ [ 2405.880700] env[67752]: value = "domain-c8" [ 2405.880700] env[67752]: _type = "ClusterComputeResource" [ 2405.880700] env[67752]: } {{(pid=67752) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2122}} [ 2405.881966] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-9596a2ae-fb43-4fcf-ad56-df0993a77b96 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2405.892029] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Got total of 3 instances {{(pid=67752) list_instances /opt/stack/nova/nova/virt/vmwareapi/vmops.py:2131}} [ 2405.892138] env[67752]: WARNING nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] While synchronizing instance power states, found 4 instances in the database and 3 instances on the hypervisor. [ 2405.892263] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid e623c04f-674f-46b8-9477-400f8421054e {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 2405.892451] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid 1a18b0c1-48e6-423d-a43c-6c07ffedbe06 {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 2405.892617] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid 985fe487-26aa-467c-8e65-c0a341fd21ab {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 2405.892782] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Triggering sync for uuid da4910c9-6710-436c-8112-1b8030286623 {{(pid=67752) _sync_power_states /opt/stack/nova/nova/compute/manager.py:10329}} [ 2405.893063] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "e623c04f-674f-46b8-9477-400f8421054e" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2405.893312] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "1a18b0c1-48e6-423d-a43c-6c07ffedbe06" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2405.893549] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "985fe487-26aa-467c-8e65-c0a341fd21ab" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2405.893782] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "da4910c9-6710-436c-8112-1b8030286623" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2405.897383] env[67752]: DEBUG nova.virt.vmwareapi.images [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Downloading image file data 5e3817d9-8178-4d2c-9c59-eb1eb0833380 to the data store datastore2 {{(pid=67752) fetch_image /opt/stack/nova/nova/virt/vmwareapi/images.py:245}} [ 2405.950237] env[67752]: DEBUG oslo_vmware.rw_handles [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Creating HTTP connection to write to file with size = 21318656 and URL = https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/f47866f3-a8c6-4abf-8e33-fb702db155d8/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) _create_write_connection /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:122}} [ 2406.011070] env[67752]: DEBUG oslo_vmware.rw_handles [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Completed reading data from the image iterator. {{(pid=67752) read /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:765}} [ 2406.011282] env[67752]: DEBUG oslo_vmware.rw_handles [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Closing write handle for https://esx7c1n1.openstack.eu-de-1.cloud.sap:443/folder/vmware_temp/f47866f3-a8c6-4abf-8e33-fb702db155d8/5e3817d9-8178-4d2c-9c59-eb1eb0833380/tmp-sparse.vmdk?dcPath=ha-datacenter&dsName=datastore2. {{(pid=67752) close /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/rw_handles.py:281}} [ 2406.036694] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7f89b320-cd61-4ab7-80d3-ede42a0ea761 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2406.049374] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-828fd546-c958-4579-83c4-baa77e787674 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2406.079912] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-daf8fad6-186a-4d6d-95f0-d65abc783642 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2406.086810] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-846f2bad-a91d-44d1-b731-f446d5fdd403 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2406.099481] env[67752]: DEBUG nova.compute.provider_tree [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2406.108058] env[67752]: DEBUG nova.scheduler.client.report [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2406.121155] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.abort_instance_claim" :: held 0.245s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2406.121661] env[67752]: ERROR nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Failed to build and run instance: oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2406.121661] env[67752]: Faults: ['InvalidArgument'] [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] Traceback (most recent call last): [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] self.driver.spawn(context, instance, image_meta, [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] self._vmops.spawn(context, instance, image_meta, injected_files, [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] self._fetch_image_if_missing(context, vi) [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] image_cache(vi, tmp_image_ds_loc) [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] vm_util.copy_virtual_disk( [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] session._wait_for_task(vmdk_copy_task) [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] return self.wait_for_task(task_ref) [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] return evt.wait() [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] result = hub.switch() [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] return self.greenlet.switch() [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] self.f(*self.args, **self.kw) [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] raise exceptions.translate_fault(task_info.error) [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] Faults: ['InvalidArgument'] [ 2406.121661] env[67752]: ERROR nova.compute.manager [instance: e623c04f-674f-46b8-9477-400f8421054e] [ 2406.122501] env[67752]: DEBUG nova.compute.utils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] VimFaultException {{(pid=67752) notify_about_instance_usage /opt/stack/nova/nova/compute/utils.py:430}} [ 2406.123729] env[67752]: DEBUG nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Build of instance e623c04f-674f-46b8-9477-400f8421054e was re-scheduled: A specified parameter was not correct: fileType [ 2406.123729] env[67752]: Faults: ['InvalidArgument'] {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2471}} [ 2406.124110] env[67752]: DEBUG nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Unplugging VIFs for instance {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:2997}} [ 2406.124306] env[67752]: DEBUG nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Virt driver does not provide unplug_vifs method, so it is not possible determine if VIFs should be unplugged. {{(pid=67752) _cleanup_allocated_networks /opt/stack/nova/nova/compute/manager.py:3020}} [ 2406.124490] env[67752]: DEBUG nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 2406.124660] env[67752]: DEBUG nova.network.neutron [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2406.400034] env[67752]: DEBUG nova.network.neutron [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2406.415957] env[67752]: INFO nova.compute.manager [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Took 0.29 seconds to deallocate network for instance. [ 2406.503820] env[67752]: INFO nova.scheduler.client.report [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Deleted allocations for instance e623c04f-674f-46b8-9477-400f8421054e [ 2406.527487] env[67752]: DEBUG oslo_concurrency.lockutils [None req-bbf1d685-215c-4a31-9875-b59463c9e070 tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "e623c04f-674f-46b8-9477-400f8421054e" "released" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: held 407.337s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2406.527790] env[67752]: DEBUG oslo_concurrency.lockutils [None req-00e0682e-8be9-4b61-81dd-2aa88b5e3edc tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "e623c04f-674f-46b8-9477-400f8421054e" acquired by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: waited 211.989s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2406.528019] env[67752]: DEBUG oslo_concurrency.lockutils [None req-00e0682e-8be9-4b61-81dd-2aa88b5e3edc tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Acquiring lock "e623c04f-674f-46b8-9477-400f8421054e-events" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2406.528234] env[67752]: DEBUG oslo_concurrency.lockutils [None req-00e0682e-8be9-4b61-81dd-2aa88b5e3edc tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "e623c04f-674f-46b8-9477-400f8421054e-events" acquired by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2406.528438] env[67752]: DEBUG oslo_concurrency.lockutils [None req-00e0682e-8be9-4b61-81dd-2aa88b5e3edc tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "e623c04f-674f-46b8-9477-400f8421054e-events" "released" by "nova.compute.manager.InstanceEvents.clear_events_for_instance.._clear_events" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2406.530322] env[67752]: INFO nova.compute.manager [None req-00e0682e-8be9-4b61-81dd-2aa88b5e3edc tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Terminating instance [ 2406.532062] env[67752]: DEBUG nova.compute.manager [None req-00e0682e-8be9-4b61-81dd-2aa88b5e3edc tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Start destroying the instance on the hypervisor. {{(pid=67752) _shutdown_instance /opt/stack/nova/nova/compute/manager.py:3141}} [ 2406.532273] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-00e0682e-8be9-4b61-81dd-2aa88b5e3edc tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Destroying instance {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1142}} [ 2406.532788] env[67752]: DEBUG oslo_vmware.service [-] Invoking SearchIndex.FindAllByUuid with opID=oslo.vmware-e651a2a2-dc7f-4b79-b9d7-e47a4b39107b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2406.541540] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-2a1f65d0-74bc-4b08-bcc6-c71aba8068b3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2406.567431] env[67752]: WARNING nova.virt.vmwareapi.vmops [None req-00e0682e-8be9-4b61-81dd-2aa88b5e3edc tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Instance does not exist on backend: nova.exception.InstanceNotFound: Instance e623c04f-674f-46b8-9477-400f8421054e could not be found. [ 2406.567658] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-00e0682e-8be9-4b61-81dd-2aa88b5e3edc tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Instance destroyed {{(pid=67752) destroy /opt/stack/nova/nova/virt/vmwareapi/vmops.py:1144}} [ 2406.567845] env[67752]: INFO nova.compute.manager [None req-00e0682e-8be9-4b61-81dd-2aa88b5e3edc tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] [instance: e623c04f-674f-46b8-9477-400f8421054e] Took 0.04 seconds to destroy the instance on the hypervisor. [ 2406.568095] env[67752]: DEBUG oslo.service.loopingcall [None req-00e0682e-8be9-4b61-81dd-2aa88b5e3edc tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Waiting for function nova.compute.manager.ComputeManager._try_deallocate_network.._deallocate_network_with_retries to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2406.568385] env[67752]: DEBUG nova.compute.manager [-] [instance: e623c04f-674f-46b8-9477-400f8421054e] Deallocating network for instance {{(pid=67752) _deallocate_network /opt/stack/nova/nova/compute/manager.py:2280}} [ 2406.568432] env[67752]: DEBUG nova.network.neutron [-] [instance: e623c04f-674f-46b8-9477-400f8421054e] deallocate_for_instance() {{(pid=67752) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 2406.591572] env[67752]: DEBUG nova.network.neutron [-] [instance: e623c04f-674f-46b8-9477-400f8421054e] Updating instance_info_cache with network_info: [] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2406.599554] env[67752]: INFO nova.compute.manager [-] [instance: e623c04f-674f-46b8-9477-400f8421054e] Took 0.03 seconds to deallocate network for instance. [ 2406.688406] env[67752]: DEBUG oslo_concurrency.lockutils [None req-00e0682e-8be9-4b61-81dd-2aa88b5e3edc tempest-MultipleCreateTestJSON-288251255 tempest-MultipleCreateTestJSON-288251255-project-member] Lock "e623c04f-674f-46b8-9477-400f8421054e" "released" by "nova.compute.manager.ComputeManager.terminate_instance..do_terminate_instance" :: held 0.160s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2406.689445] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "e623c04f-674f-46b8-9477-400f8421054e" acquired by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: waited 0.796s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2406.689445] env[67752]: INFO nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: e623c04f-674f-46b8-9477-400f8421054e] During sync_power_state the instance has a pending task (deleting). Skip. [ 2406.689445] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "e623c04f-674f-46b8-9477-400f8421054e" "released" by "nova.compute.manager.ComputeManager._sync_power_states.._sync..query_driver_power_state_and_sync" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2413.660743] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._check_instance_build_time {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2415.634327] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._instance_usage_audit {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2416.635014] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_volume_usage {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2417.635605] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_unconfirmed_resizes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2420.634717] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rebooting_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2421.634817] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._reclaim_queued_deletes {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2421.635213] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] CONF.reclaim_instance_interval <= 0, skipping... {{(pid=67752) _reclaim_queued_deletes /opt/stack/nova/nova/compute/manager.py:10538}} [ 2422.635764] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._heal_instance_info_cache {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2422.636729] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Starting heal instance info cache {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9919}} [ 2422.637111] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Rebuilding the list of instances to heal {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9923}} [ 2422.663061] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 1a18b0c1-48e6-423d-a43c-6c07ffedbe06] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2422.663061] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: 985fe487-26aa-467c-8e65-c0a341fd21ab] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2422.663265] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] [instance: da4910c9-6710-436c-8112-1b8030286623] Skipping network cache update for instance because it is Building. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:9932}} [ 2422.663265] env[67752]: DEBUG nova.compute.manager [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Didn't find any instances for network info cache update. {{(pid=67752) _heal_instance_info_cache /opt/stack/nova/nova/compute/manager.py:10005}} [ 2424.635017] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager._poll_rescued_instances {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2429.634952] env[67752]: DEBUG oslo_service.periodic_task [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Running periodic task ComputeManager.update_available_resource {{(pid=67752) run_periodic_tasks /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/periodic_task.py:210}} [ 2429.646749] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2429.646986] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2429.647181] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.clean_compute_node_cache" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2429.647346] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Auditing locally available compute resources for cpu-1 (node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28) {{(pid=67752) update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:907}} [ 2429.648471] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4695ed62-7102-40ef-ab9e-714ebd60dcba {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2429.657752] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5d156805-e1ce-4897-92ac-be9ff376c0e3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2429.671908] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-5866178a-8a94-44b4-9eff-476dba23a978 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2429.678052] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-647a5404-daba-44a2-96cb-d089e6e8a9bf {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2429.707011] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Hypervisor/Node resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 free_ram=180996MB free_disk=126GB free_vcpus=48 pci_devices=None {{(pid=67752) _report_hypervisor_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1106}} [ 2429.707165] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2429.707353] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2429.757924] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 1a18b0c1-48e6-423d-a43c-6c07ffedbe06 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2429.758183] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance 985fe487-26aa-467c-8e65-c0a341fd21ab actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2429.758317] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Instance da4910c9-6710-436c-8112-1b8030286623 actively managed on this compute host and has allocations in placement: {'resources': {'DISK_GB': 1, 'MEMORY_MB': 128, 'VCPU': 1}}. {{(pid=67752) _remove_deleted_instances_allocations /opt/stack/nova/nova/compute/resource_tracker.py:1707}} [ 2429.758490] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Total usable vcpus: 48, total allocated vcpus: 3 {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1129}} [ 2429.758632] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Final resource view: name=domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 phys_ram=196590MB used_ram=896MB phys_disk=200GB used_disk=3GB total_vcpus=48 used_vcpus=3 pci_stats=[] {{(pid=67752) _report_final_resource_view /opt/stack/nova/nova/compute/resource_tracker.py:1138}} [ 2429.806581] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-754ad69d-2c99-4237-a87f-132a0e7d7cef {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2429.813601] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-a65ae4d2-9029-4516-bf03-f5687bd4e0b1 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2429.842384] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-1dbaf7ea-466d-4b1b-b577-58bf32753bb7 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2429.848978] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-69bce816-fe42-48c7-b148-ef7931a18b8f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2429.861583] env[67752]: DEBUG nova.compute.provider_tree [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2429.870137] env[67752]: DEBUG nova.scheduler.client.report [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2429.882995] env[67752]: DEBUG nova.compute.resource_tracker [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Compute_service record updated for cpu-1:domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 {{(pid=67752) _update_available_resource /opt/stack/nova/nova/compute/resource_tracker.py:1067}} [ 2429.883196] env[67752]: DEBUG oslo_concurrency.lockutils [None req-9a59c0ab-8a95-4f71-a076-487a0e7c39c8 None None] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker._update_available_resource" :: held 0.176s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2435.196567] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "90bb48c2-530e-4da0-8047-1ed4069d67f4" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2435.196885] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "90bb48c2-530e-4da0-8047-1ed4069d67f4" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2435.207131] env[67752]: DEBUG nova.compute.manager [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 2435.257024] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2435.257281] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2435.258746] env[67752]: INFO nova.compute.claims [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2435.362170] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ac39463d-4b69-4387-8593-17ae55fdb085 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2435.370898] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b1ba60a4-ac02-43c6-ba61-daee177d340b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2435.403826] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0f3423c2-3bdb-433e-9320-9f89818aafbe {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2435.411057] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-e7509a03-1cc2-40d9-9e1e-8ad9c77f8bbd {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2435.424853] env[67752]: DEBUG nova.compute.provider_tree [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2435.434535] env[67752]: DEBUG nova.scheduler.client.report [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2435.448952] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.192s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2435.449425] env[67752]: DEBUG nova.compute.manager [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 2435.480388] env[67752]: DEBUG nova.compute.utils [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2435.481593] env[67752]: DEBUG nova.compute.manager [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 2435.481782] env[67752]: DEBUG nova.network.neutron [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 2435.490444] env[67752]: DEBUG nova.compute.manager [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 2435.564421] env[67752]: DEBUG nova.policy [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '7ceddc353fd444808a8e3fa9a98925c8', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': '17776dc4e25145179ebfbdfd4a9639f8', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 2435.569842] env[67752]: DEBUG nova.compute.manager [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 2435.604534] env[67752]: DEBUG nova.virt.hardware [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 2435.604818] env[67752]: DEBUG nova.virt.hardware [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 2435.604990] env[67752]: DEBUG nova.virt.hardware [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2435.605218] env[67752]: DEBUG nova.virt.hardware [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 2435.605381] env[67752]: DEBUG nova.virt.hardware [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2435.605547] env[67752]: DEBUG nova.virt.hardware [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 2435.605757] env[67752]: DEBUG nova.virt.hardware [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 2435.605938] env[67752]: DEBUG nova.virt.hardware [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 2435.606137] env[67752]: DEBUG nova.virt.hardware [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 2435.606320] env[67752]: DEBUG nova.virt.hardware [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 2435.606516] env[67752]: DEBUG nova.virt.hardware [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 2435.607723] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-fc99bf56-b4bb-4a7d-887a-9cedaba283fe {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2435.616143] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-54e5de92-6c20-4a0d-8a04-845eb1716ce5 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2435.896133] env[67752]: DEBUG nova.network.neutron [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Successfully created port: f8bbfc65-e465-4c99-b14b-ffa41ddfc27a {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2436.517063] env[67752]: DEBUG nova.compute.manager [req-f2f584c2-a517-49c2-a135-9a9fb51cee3c req-0d851dfe-f31a-4df2-b096-2d43ccd73d0c service nova] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Received event network-vif-plugged-f8bbfc65-e465-4c99-b14b-ffa41ddfc27a {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 2436.517335] env[67752]: DEBUG oslo_concurrency.lockutils [req-f2f584c2-a517-49c2-a135-9a9fb51cee3c req-0d851dfe-f31a-4df2-b096-2d43ccd73d0c service nova] Acquiring lock "90bb48c2-530e-4da0-8047-1ed4069d67f4-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2436.517501] env[67752]: DEBUG oslo_concurrency.lockutils [req-f2f584c2-a517-49c2-a135-9a9fb51cee3c req-0d851dfe-f31a-4df2-b096-2d43ccd73d0c service nova] Lock "90bb48c2-530e-4da0-8047-1ed4069d67f4-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2436.517698] env[67752]: DEBUG oslo_concurrency.lockutils [req-f2f584c2-a517-49c2-a135-9a9fb51cee3c req-0d851dfe-f31a-4df2-b096-2d43ccd73d0c service nova] Lock "90bb48c2-530e-4da0-8047-1ed4069d67f4-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2436.517873] env[67752]: DEBUG nova.compute.manager [req-f2f584c2-a517-49c2-a135-9a9fb51cee3c req-0d851dfe-f31a-4df2-b096-2d43ccd73d0c service nova] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] No waiting events found dispatching network-vif-plugged-f8bbfc65-e465-4c99-b14b-ffa41ddfc27a {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 2436.518049] env[67752]: WARNING nova.compute.manager [req-f2f584c2-a517-49c2-a135-9a9fb51cee3c req-0d851dfe-f31a-4df2-b096-2d43ccd73d0c service nova] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Received unexpected event network-vif-plugged-f8bbfc65-e465-4c99-b14b-ffa41ddfc27a for instance with vm_state building and task_state spawning. [ 2436.603585] env[67752]: DEBUG nova.network.neutron [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Successfully updated port: f8bbfc65-e465-4c99-b14b-ffa41ddfc27a {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 2436.616981] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "refresh_cache-90bb48c2-530e-4da0-8047-1ed4069d67f4" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2436.616981] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquired lock "refresh_cache-90bb48c2-530e-4da0-8047-1ed4069d67f4" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2436.616981] env[67752]: DEBUG nova.network.neutron [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 2436.660829] env[67752]: DEBUG nova.network.neutron [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 2437.018735] env[67752]: DEBUG nova.network.neutron [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Updating instance_info_cache with network_info: [{"id": "f8bbfc65-e465-4c99-b14b-ffa41ddfc27a", "address": "fa:16:3e:02:87:14", "network": {"id": "0fbb58d7-f336-4105-b40d-14fe23cb5231", "bridge": "br-int", "label": "tempest-ServersTestJSON-455316140-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "17776dc4e25145179ebfbdfd4a9639f8", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "92cdccfd-4b10-4024-b724-5f22792dd4de", "external-id": "nsx-vlan-transportzone-902", "segmentation_id": 902, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapf8bbfc65-e4", "ovs_interfaceid": "f8bbfc65-e465-4c99-b14b-ffa41ddfc27a", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2437.029571] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Releasing lock "refresh_cache-90bb48c2-530e-4da0-8047-1ed4069d67f4" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2437.029849] env[67752]: DEBUG nova.compute.manager [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Instance network_info: |[{"id": "f8bbfc65-e465-4c99-b14b-ffa41ddfc27a", "address": "fa:16:3e:02:87:14", "network": {"id": "0fbb58d7-f336-4105-b40d-14fe23cb5231", "bridge": "br-int", "label": "tempest-ServersTestJSON-455316140-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "17776dc4e25145179ebfbdfd4a9639f8", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "92cdccfd-4b10-4024-b724-5f22792dd4de", "external-id": "nsx-vlan-transportzone-902", "segmentation_id": 902, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapf8bbfc65-e4", "ovs_interfaceid": "f8bbfc65-e465-4c99-b14b-ffa41ddfc27a", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 2437.030294] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:02:87:14', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': '92cdccfd-4b10-4024-b724-5f22792dd4de', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': 'f8bbfc65-e465-4c99-b14b-ffa41ddfc27a', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 2437.037969] env[67752]: DEBUG oslo.service.loopingcall [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2437.038594] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 2437.039216] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-a11cd6db-1a92-4e45-b142-972c3ae41c56 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2437.058684] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 2437.058684] env[67752]: value = "task-3199865" [ 2437.058684] env[67752]: _type = "Task" [ 2437.058684] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2437.066056] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199865, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2437.568753] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199865, 'name': CreateVM_Task, 'duration_secs': 0.285636} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2437.569125] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 2437.569612] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2437.569782] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2437.570133] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 2437.570369] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-fe4298d3-2773-4888-9889-936405abbb9b {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2437.574454] env[67752]: DEBUG oslo_vmware.api [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Waiting for the task: (returnval){ [ 2437.574454] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5235c4fd-e7c1-6dfc-1649-1c04218c5a17" [ 2437.574454] env[67752]: _type = "Task" [ 2437.574454] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2437.581592] env[67752]: DEBUG oslo_vmware.api [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]5235c4fd-e7c1-6dfc-1649-1c04218c5a17, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2438.085478] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2438.085732] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 2438.085946] env[67752]: DEBUG oslo_concurrency.lockutils [None req-f97dfdff-65f7-44c6-80b9-151340e9b06a tempest-ServersTestJSON-1675883982 tempest-ServersTestJSON-1675883982-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2438.596198] env[67752]: DEBUG nova.compute.manager [req-aa2a8866-41cf-4de6-9691-4ba3e49f5d21 req-ba4e917d-cef3-479c-9fea-998092951126 service nova] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Received event network-changed-f8bbfc65-e465-4c99-b14b-ffa41ddfc27a {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 2438.596446] env[67752]: DEBUG nova.compute.manager [req-aa2a8866-41cf-4de6-9691-4ba3e49f5d21 req-ba4e917d-cef3-479c-9fea-998092951126 service nova] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Refreshing instance network info cache due to event network-changed-f8bbfc65-e465-4c99-b14b-ffa41ddfc27a. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 2438.596668] env[67752]: DEBUG oslo_concurrency.lockutils [req-aa2a8866-41cf-4de6-9691-4ba3e49f5d21 req-ba4e917d-cef3-479c-9fea-998092951126 service nova] Acquiring lock "refresh_cache-90bb48c2-530e-4da0-8047-1ed4069d67f4" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2438.596843] env[67752]: DEBUG oslo_concurrency.lockutils [req-aa2a8866-41cf-4de6-9691-4ba3e49f5d21 req-ba4e917d-cef3-479c-9fea-998092951126 service nova] Acquired lock "refresh_cache-90bb48c2-530e-4da0-8047-1ed4069d67f4" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2438.597016] env[67752]: DEBUG nova.network.neutron [req-aa2a8866-41cf-4de6-9691-4ba3e49f5d21 req-ba4e917d-cef3-479c-9fea-998092951126 service nova] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Refreshing network info cache for port f8bbfc65-e465-4c99-b14b-ffa41ddfc27a {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 2438.840259] env[67752]: DEBUG nova.network.neutron [req-aa2a8866-41cf-4de6-9691-4ba3e49f5d21 req-ba4e917d-cef3-479c-9fea-998092951126 service nova] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Updated VIF entry in instance network info cache for port f8bbfc65-e465-4c99-b14b-ffa41ddfc27a. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 2438.840688] env[67752]: DEBUG nova.network.neutron [req-aa2a8866-41cf-4de6-9691-4ba3e49f5d21 req-ba4e917d-cef3-479c-9fea-998092951126 service nova] [instance: 90bb48c2-530e-4da0-8047-1ed4069d67f4] Updating instance_info_cache with network_info: [{"id": "f8bbfc65-e465-4c99-b14b-ffa41ddfc27a", "address": "fa:16:3e:02:87:14", "network": {"id": "0fbb58d7-f336-4105-b40d-14fe23cb5231", "bridge": "br-int", "label": "tempest-ServersTestJSON-455316140-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.14", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "17776dc4e25145179ebfbdfd4a9639f8", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "92cdccfd-4b10-4024-b724-5f22792dd4de", "external-id": "nsx-vlan-transportzone-902", "segmentation_id": 902, "bound_drivers": {"0": "nsxv3"}}, "devname": "tapf8bbfc65-e4", "ovs_interfaceid": "f8bbfc65-e465-4c99-b14b-ffa41ddfc27a", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2438.849890] env[67752]: DEBUG oslo_concurrency.lockutils [req-aa2a8866-41cf-4de6-9691-4ba3e49f5d21 req-ba4e917d-cef3-479c-9fea-998092951126 service nova] Releasing lock "refresh_cache-90bb48c2-530e-4da0-8047-1ed4069d67f4" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2448.323188] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Acquiring lock "311eb104-7078-40b5-9735-195911a77d2d" by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2448.323505] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Lock "311eb104-7078-40b5-9735-195911a77d2d" acquired by "nova.compute.manager.ComputeManager.build_and_run_instance.._locked_do_build_and_run_instance" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2448.335497] env[67752]: DEBUG nova.compute.manager [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Starting instance... {{(pid=67752) _do_build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2423}} [ 2448.385036] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Acquiring lock "compute_resources" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2448.385299] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Lock "compute_resources" acquired by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2448.386706] env[67752]: INFO nova.compute.claims [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Claim successful on node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28 [ 2448.491141] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-4ca63584-68b5-477f-8533-7d0b3e2610d3 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2448.499163] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-b64e38d8-2f44-4ad0-8b05-ece1f35ed558 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2448.527972] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-65de28c6-9340-4907-8196-8759985d10e9 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2448.534441] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-7b0955a5-94c7-416e-9f01-ecf0be45fa61 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2448.548016] env[67752]: DEBUG nova.compute.provider_tree [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Inventory has not changed in ProviderTree for provider: c7141b98-bb6c-46bd-b37f-a8e913940199 {{(pid=67752) update_inventory /opt/stack/nova/nova/compute/provider_tree.py:180}} [ 2448.556545] env[67752]: DEBUG nova.scheduler.client.report [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Inventory has not changed for provider c7141b98-bb6c-46bd-b37f-a8e913940199 based on inventory data: {'VCPU': {'total': 48, 'reserved': 0, 'min_unit': 1, 'max_unit': 16, 'step_size': 1, 'allocation_ratio': 4.0}, 'MEMORY_MB': {'total': 196590, 'reserved': 512, 'min_unit': 1, 'max_unit': 65530, 'step_size': 1, 'allocation_ratio': 1.0}, 'DISK_GB': {'total': 400, 'reserved': 0, 'min_unit': 1, 'max_unit': 126, 'step_size': 1, 'allocation_ratio': 1.0}} {{(pid=67752) set_inventory_for_provider /opt/stack/nova/nova/scheduler/client/report.py:940}} [ 2448.569836] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Lock "compute_resources" "released" by "nova.compute.resource_tracker.ResourceTracker.instance_claim" :: held 0.184s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2448.569836] env[67752]: DEBUG nova.compute.manager [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Start building networks asynchronously for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2820}} [ 2448.599850] env[67752]: DEBUG nova.compute.utils [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Using /dev/sd instead of None {{(pid=67752) get_next_device_name /opt/stack/nova/nova/compute/utils.py:238}} [ 2448.601032] env[67752]: DEBUG nova.compute.manager [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Allocating IP information in the background. {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1972}} [ 2448.601225] env[67752]: DEBUG nova.network.neutron [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: 311eb104-7078-40b5-9735-195911a77d2d] allocate_for_instance() {{(pid=67752) allocate_for_instance /opt/stack/nova/nova/network/neutron.py:1156}} [ 2448.611828] env[67752]: DEBUG nova.compute.manager [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Start building block device mappings for instance. {{(pid=67752) _build_resources /opt/stack/nova/nova/compute/manager.py:2855}} [ 2448.672855] env[67752]: DEBUG nova.policy [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Policy check for network:attach_external_network failed with credentials {'is_admin': False, 'user_id': '40a6c9b19bc14ed0b4e1b5b3b031c692', 'user_domain_id': 'default', 'system_scope': None, 'domain_id': None, 'project_id': 'f7d3fb6872294f31bd9fa2b48f19808f', 'project_domain_id': 'default', 'roles': ['member', 'reader'], 'is_admin_project': True, 'service_user_id': None, 'service_user_domain_id': None, 'service_project_id': None, 'service_project_domain_id': None, 'service_roles': []} {{(pid=67752) authorize /opt/stack/nova/nova/policy.py:203}} [ 2448.680893] env[67752]: DEBUG nova.compute.manager [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Start spawning the instance on the hypervisor. {{(pid=67752) _build_and_run_instance /opt/stack/nova/nova/compute/manager.py:2629}} [ 2448.704079] env[67752]: DEBUG nova.virt.hardware [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Getting desirable topologies for flavor Flavor(created_at=2025-03-27T04:31:49Z,deleted=False,deleted_at=None,description=None,disabled=False,ephemeral_gb=0,extra_specs={hw_rng:allowed='True'},flavorid='42',id=11,is_public=True,memory_mb=128,name='m1.nano',projects=,root_gb=1,rxtx_factor=1.0,swap=0,updated_at=None,vcpu_weight=0,vcpus=1) and image_meta ImageMeta(checksum='879b8eacf4b511d84bac79c7fe0e0d0a',container_format='bare',created_at=2025-03-27T04:31:34Z,direct_url=,disk_format='vmdk',id=5e3817d9-8178-4d2c-9c59-eb1eb0833380,min_disk=0,min_ram=0,name='cirros-d240228-sparse;paraVirtual;vmxnet3',owner='c43e0a2618b34312898e0e4bb014b17c',properties=ImageMetaProps,protected=,size=21318656,status='active',tags=,updated_at=2025-03-27T04:31:35Z,virtual_size=,visibility=), allow threads: False {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:563}} [ 2448.704336] env[67752]: DEBUG nova.virt.hardware [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Flavor limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:348}} [ 2448.704502] env[67752]: DEBUG nova.virt.hardware [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Image limits 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:352}} [ 2448.704690] env[67752]: DEBUG nova.virt.hardware [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Flavor pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:388}} [ 2448.704838] env[67752]: DEBUG nova.virt.hardware [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Image pref 0:0:0 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:392}} [ 2448.704987] env[67752]: DEBUG nova.virt.hardware [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Chose sockets=0, cores=0, threads=0; limits were sockets=65536, cores=65536, threads=65536 {{(pid=67752) get_cpu_topology_constraints /opt/stack/nova/nova/virt/hardware.py:430}} [ 2448.705215] env[67752]: DEBUG nova.virt.hardware [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Topology preferred VirtCPUTopology(cores=0,sockets=0,threads=0), maximum VirtCPUTopology(cores=65536,sockets=65536,threads=65536) {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:569}} [ 2448.705378] env[67752]: DEBUG nova.virt.hardware [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Build topologies for 1 vcpu(s) 1:1:1 {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:471}} [ 2448.705547] env[67752]: DEBUG nova.virt.hardware [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Got 1 possible topologies {{(pid=67752) _get_possible_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:501}} [ 2448.705711] env[67752]: DEBUG nova.virt.hardware [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Possible topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:575}} [ 2448.705886] env[67752]: DEBUG nova.virt.hardware [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Sorted desired topologies [VirtCPUTopology(cores=1,sockets=1,threads=1)] {{(pid=67752) _get_desirable_cpu_topologies /opt/stack/nova/nova/virt/hardware.py:577}} [ 2448.706754] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-0d55cbc2-81e6-4e6e-a0c0-202b0151ad8f {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2448.714507] env[67752]: DEBUG oslo_vmware.service [-] Invoking PropertyCollector.RetrievePropertiesEx with opID=oslo.vmware-ddbfc9f4-f895-4c87-a9f4-73a52dd45fa2 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2449.012503] env[67752]: DEBUG nova.network.neutron [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Successfully created port: 1ce899e9-43dc-4c3e-a928-c9d38424dc41 {{(pid=67752) _create_port_minimal /opt/stack/nova/nova/network/neutron.py:548}} [ 2449.542838] env[67752]: DEBUG nova.compute.manager [req-6a438d94-2b1d-418d-a749-5de7bd4be18c req-afc5fa54-2b12-4c5f-97fc-0095c03f4be8 service nova] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Received event network-vif-plugged-1ce899e9-43dc-4c3e-a928-c9d38424dc41 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 2449.543090] env[67752]: DEBUG oslo_concurrency.lockutils [req-6a438d94-2b1d-418d-a749-5de7bd4be18c req-afc5fa54-2b12-4c5f-97fc-0095c03f4be8 service nova] Acquiring lock "311eb104-7078-40b5-9735-195911a77d2d-events" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 2449.543531] env[67752]: DEBUG oslo_concurrency.lockutils [req-6a438d94-2b1d-418d-a749-5de7bd4be18c req-afc5fa54-2b12-4c5f-97fc-0095c03f4be8 service nova] Lock "311eb104-7078-40b5-9735-195911a77d2d-events" acquired by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: waited 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 2449.543531] env[67752]: DEBUG oslo_concurrency.lockutils [req-6a438d94-2b1d-418d-a749-5de7bd4be18c req-afc5fa54-2b12-4c5f-97fc-0095c03f4be8 service nova] Lock "311eb104-7078-40b5-9735-195911a77d2d-events" "released" by "nova.compute.manager.InstanceEvents.pop_instance_event.._pop_event" :: held 0.000s {{(pid=67752) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 2449.543681] env[67752]: DEBUG nova.compute.manager [req-6a438d94-2b1d-418d-a749-5de7bd4be18c req-afc5fa54-2b12-4c5f-97fc-0095c03f4be8 service nova] [instance: 311eb104-7078-40b5-9735-195911a77d2d] No waiting events found dispatching network-vif-plugged-1ce899e9-43dc-4c3e-a928-c9d38424dc41 {{(pid=67752) pop_instance_event /opt/stack/nova/nova/compute/manager.py:320}} [ 2449.543848] env[67752]: WARNING nova.compute.manager [req-6a438d94-2b1d-418d-a749-5de7bd4be18c req-afc5fa54-2b12-4c5f-97fc-0095c03f4be8 service nova] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Received unexpected event network-vif-plugged-1ce899e9-43dc-4c3e-a928-c9d38424dc41 for instance with vm_state building and task_state spawning. [ 2449.626517] env[67752]: DEBUG nova.network.neutron [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Successfully updated port: 1ce899e9-43dc-4c3e-a928-c9d38424dc41 {{(pid=67752) _update_port /opt/stack/nova/nova/network/neutron.py:586}} [ 2449.639882] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Acquiring lock "refresh_cache-311eb104-7078-40b5-9735-195911a77d2d" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2449.640072] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Acquired lock "refresh_cache-311eb104-7078-40b5-9735-195911a77d2d" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2449.640249] env[67752]: DEBUG nova.network.neutron [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Building network info cache for instance {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2010}} [ 2449.684618] env[67752]: DEBUG nova.network.neutron [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Instance cache missing network info. {{(pid=67752) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 2449.844646] env[67752]: DEBUG nova.network.neutron [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Updating instance_info_cache with network_info: [{"id": "1ce899e9-43dc-4c3e-a928-c9d38424dc41", "address": "fa:16:3e:28:33:42", "network": {"id": "bef55da3-1050-4c26-9415-aef048e94b8c", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-327762555-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "f7d3fb6872294f31bd9fa2b48f19808f", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "d6e940e5-e083-4238-973e-f1b4e2a3a5c7", "external-id": "nsx-vlan-transportzone-64", "segmentation_id": 64, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap1ce899e9-43", "ovs_interfaceid": "1ce899e9-43dc-4c3e-a928-c9d38424dc41", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2449.855531] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Releasing lock "refresh_cache-311eb104-7078-40b5-9735-195911a77d2d" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2449.855820] env[67752]: DEBUG nova.compute.manager [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Instance network_info: |[{"id": "1ce899e9-43dc-4c3e-a928-c9d38424dc41", "address": "fa:16:3e:28:33:42", "network": {"id": "bef55da3-1050-4c26-9415-aef048e94b8c", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-327762555-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "f7d3fb6872294f31bd9fa2b48f19808f", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "d6e940e5-e083-4238-973e-f1b4e2a3a5c7", "external-id": "nsx-vlan-transportzone-64", "segmentation_id": 64, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap1ce899e9-43", "ovs_interfaceid": "1ce899e9-43dc-4c3e-a928-c9d38424dc41", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}]| {{(pid=67752) _allocate_network_async /opt/stack/nova/nova/compute/manager.py:1987}} [ 2449.856242] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Instance VIF info [{'network_name': 'br-int', 'mac_address': 'fa:16:3e:28:33:42', 'network_ref': {'type': 'OpaqueNetwork', 'network-id': 'd6e940e5-e083-4238-973e-f1b4e2a3a5c7', 'network-type': 'nsx.LogicalSwitch', 'use-external-id': True}, 'iface_id': '1ce899e9-43dc-4c3e-a928-c9d38424dc41', 'vif_model': 'vmxnet3'}] {{(pid=67752) build_virtual_machine /opt/stack/nova/nova/virt/vmwareapi/vmops.py:279}} [ 2449.863904] env[67752]: DEBUG oslo.service.loopingcall [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Waiting for function nova.virt.vmwareapi.vm_util.create_vm to return. {{(pid=67752) func /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/loopingcall.py:435}} [ 2449.864375] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Creating VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1327}} [ 2449.864601] env[67752]: DEBUG oslo_vmware.service [-] Invoking Folder.CreateVM_Task with opID=oslo.vmware-c817ffb1-661c-4065-a195-c011ccf913c6 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2449.886933] env[67752]: DEBUG oslo_vmware.api [-] Waiting for the task: (returnval){ [ 2449.886933] env[67752]: value = "task-3199866" [ 2449.886933] env[67752]: _type = "Task" [ 2449.886933] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2449.893985] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199866, 'name': CreateVM_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2450.396963] env[67752]: DEBUG oslo_vmware.api [-] Task: {'id': task-3199866, 'name': CreateVM_Task, 'duration_secs': 0.271213} completed successfully. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:444}} [ 2450.397153] env[67752]: DEBUG nova.virt.vmwareapi.vm_util [-] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Created VM on the ESX host {{(pid=67752) create_vm /opt/stack/nova/nova/virt/vmwareapi/vm_util.py:1349}} [ 2450.397818] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2450.397986] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Acquired lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2450.398319] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Acquired external semaphore "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:321}} [ 2450.398580] env[67752]: DEBUG oslo_vmware.service [-] Invoking HostDatastoreBrowser.SearchDatastore_Task with opID=oslo.vmware-ee652c8b-abab-4d23-b755-87f10a086686 {{(pid=67752) request_handler /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/service.py:371}} [ 2450.402884] env[67752]: DEBUG oslo_vmware.api [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Waiting for the task: (returnval){ [ 2450.402884] env[67752]: value = "session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]523768eb-7565-7760-a5d8-30a02c4846d3" [ 2450.402884] env[67752]: _type = "Task" [ 2450.402884] env[67752]: } to complete. {{(pid=67752) wait_for_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:397}} [ 2450.411432] env[67752]: DEBUG oslo_vmware.api [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Task: {'id': session[52d03e2c-4f6d-537b-655a-64d75c53d3f6]523768eb-7565-7760-a5d8-30a02c4846d3, 'name': SearchDatastore_Task} progress is 0%. {{(pid=67752) _poll_task /opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py:434}} [ 2450.913512] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Releasing lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 2450.913879] env[67752]: DEBUG nova.virt.vmwareapi.vmops [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Processing image 5e3817d9-8178-4d2c-9c59-eb1eb0833380 {{(pid=67752) _fetch_image_if_missing /opt/stack/nova/nova/virt/vmwareapi/vmops.py:624}} [ 2450.913944] env[67752]: DEBUG oslo_concurrency.lockutils [None req-c27fe10d-2061-404d-9171-6a0d80b54223 tempest-DeleteServersTestJSON-598835925 tempest-DeleteServersTestJSON-598835925-project-member] Acquiring lock "[datastore2] devstack-image-cache_base/5e3817d9-8178-4d2c-9c59-eb1eb0833380/5e3817d9-8178-4d2c-9c59-eb1eb0833380.vmdk" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2451.568563] env[67752]: DEBUG nova.compute.manager [req-168542a1-cc05-4bcc-afdc-95e276b62746 req-7aa95105-47b5-44ea-90be-cb00ba09064e service nova] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Received event network-changed-1ce899e9-43dc-4c3e-a928-c9d38424dc41 {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11122}} [ 2451.568778] env[67752]: DEBUG nova.compute.manager [req-168542a1-cc05-4bcc-afdc-95e276b62746 req-7aa95105-47b5-44ea-90be-cb00ba09064e service nova] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Refreshing instance network info cache due to event network-changed-1ce899e9-43dc-4c3e-a928-c9d38424dc41. {{(pid=67752) external_instance_event /opt/stack/nova/nova/compute/manager.py:11127}} [ 2451.568998] env[67752]: DEBUG oslo_concurrency.lockutils [req-168542a1-cc05-4bcc-afdc-95e276b62746 req-7aa95105-47b5-44ea-90be-cb00ba09064e service nova] Acquiring lock "refresh_cache-311eb104-7078-40b5-9735-195911a77d2d" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 2451.569161] env[67752]: DEBUG oslo_concurrency.lockutils [req-168542a1-cc05-4bcc-afdc-95e276b62746 req-7aa95105-47b5-44ea-90be-cb00ba09064e service nova] Acquired lock "refresh_cache-311eb104-7078-40b5-9735-195911a77d2d" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 2451.569325] env[67752]: DEBUG nova.network.neutron [req-168542a1-cc05-4bcc-afdc-95e276b62746 req-7aa95105-47b5-44ea-90be-cb00ba09064e service nova] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Refreshing network info cache for port 1ce899e9-43dc-4c3e-a928-c9d38424dc41 {{(pid=67752) _get_instance_nw_info /opt/stack/nova/nova/network/neutron.py:2007}} [ 2451.809842] env[67752]: DEBUG nova.network.neutron [req-168542a1-cc05-4bcc-afdc-95e276b62746 req-7aa95105-47b5-44ea-90be-cb00ba09064e service nova] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Updated VIF entry in instance network info cache for port 1ce899e9-43dc-4c3e-a928-c9d38424dc41. {{(pid=67752) _build_network_info_model /opt/stack/nova/nova/network/neutron.py:3482}} [ 2451.810317] env[67752]: DEBUG nova.network.neutron [req-168542a1-cc05-4bcc-afdc-95e276b62746 req-7aa95105-47b5-44ea-90be-cb00ba09064e service nova] [instance: 311eb104-7078-40b5-9735-195911a77d2d] Updating instance_info_cache with network_info: [{"id": "1ce899e9-43dc-4c3e-a928-c9d38424dc41", "address": "fa:16:3e:28:33:42", "network": {"id": "bef55da3-1050-4c26-9415-aef048e94b8c", "bridge": "br-int", "label": "tempest-DeleteServersTestJSON-327762555-network", "subnets": [{"cidr": "192.168.128.0/28", "dns": [], "gateway": {"address": "192.168.128.1", "type": "gateway", "version": 4, "meta": {}}, "ips": [{"address": "192.168.128.5", "type": "fixed", "version": 4, "meta": {}, "floating_ips": []}], "routes": [], "version": 4, "meta": {"enable_dhcp": true, "dhcp_server": "192.168.128.2"}}], "meta": {"injected": false, "tenant_id": "f7d3fb6872294f31bd9fa2b48f19808f", "mtu": 8950, "physical_network": "default", "tunneled": false}}, "type": "ovs", "details": {"connectivity": "l2", "port_filter": true, "nsx-logical-switch-id": "d6e940e5-e083-4238-973e-f1b4e2a3a5c7", "external-id": "nsx-vlan-transportzone-64", "segmentation_id": 64, "bound_drivers": {"0": "nsxv3"}}, "devname": "tap1ce899e9-43", "ovs_interfaceid": "1ce899e9-43dc-4c3e-a928-c9d38424dc41", "qbh_params": null, "qbg_params": null, "active": true, "vnic_type": "normal", "profile": {}, "preserve_on_delete": false, "delegate_create": true, "meta": {}}] {{(pid=67752) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 2451.819688] env[67752]: DEBUG oslo_concurrency.lockutils [req-168542a1-cc05-4bcc-afdc-95e276b62746 req-7aa95105-47b5-44ea-90be-cb00ba09064e service nova] Releasing lock "refresh_cache-311eb104-7078-40b5-9735-195911a77d2d" {{(pid=67752) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}}