[ 394.621003] nova-conductor[53971]: INFO dbcounter [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] Registered counter for database nova_api [ 394.630002] nova-conductor[53971]: DEBUG oslo_db.sqlalchemy.engines [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=53971) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 394.659139] nova-conductor[53971]: DEBUG dbcounter [-] [53971] Writer thread running {{(pid=53971) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 394.661207] nova-conductor[53971]: DEBUG nova.context [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] Found 2 cells: 00000000-0000-0000-0000-000000000000(cell0),a4379ae1-47d8-4924-9d8e-3d761d765452(cell1) {{(pid=53971) load_cells /opt/stack/nova/nova/context.py:474}} [ 394.662915] nova-conductor[53971]: DEBUG oslo_concurrency.lockutils [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53971) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 394.663122] nova-conductor[53971]: DEBUG oslo_concurrency.lockutils [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53971) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 394.663543] nova-conductor[53971]: DEBUG oslo_concurrency.lockutils [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53971) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 394.663936] nova-conductor[53971]: DEBUG oslo_concurrency.lockutils [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53971) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 394.664142] nova-conductor[53971]: DEBUG oslo_concurrency.lockutils [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53971) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 394.665104] nova-conductor[53971]: DEBUG oslo_concurrency.lockutils [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53971) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 394.666339] nova-conductor[53971]: INFO dbcounter [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] Registered counter for database nova_cell0 [ 394.668648] nova-conductor[53971]: INFO dbcounter [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] Registered counter for database nova_cell1 [ 394.671284] nova-conductor[53971]: DEBUG oslo_db.sqlalchemy.engines [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=53971) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 394.671624] nova-conductor[53971]: DEBUG oslo_db.sqlalchemy.engines [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=53971) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 394.749115] nova-conductor[53971]: DEBUG dbcounter [-] [53971] Writer thread running {{(pid=53971) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 394.752215] nova-conductor[53971]: DEBUG dbcounter [-] [53971] Writer thread running {{(pid=53971) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 394.753827] nova-conductor[53971]: DEBUG oslo_concurrency.lockutils [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] Acquiring lock "singleton_lock" {{(pid=53971) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 394.754152] nova-conductor[53971]: DEBUG oslo_concurrency.lockutils [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] Acquired lock "singleton_lock" {{(pid=53971) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 394.754218] nova-conductor[53971]: DEBUG oslo_concurrency.lockutils [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] Releasing lock "singleton_lock" {{(pid=53971) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 394.754639] nova-conductor[53971]: INFO oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] Starting 2 workers [ 394.758722] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] Started child 54393 {{(pid=53971) _start_child /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/service.py:579}} [ 394.764406] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] Started child 54394 {{(pid=53971) _start_child /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/service.py:579}} [ 394.764536] nova-conductor[54393]: INFO nova.service [-] Starting conductor node (version 0.0.1) [ 394.764763] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] Full set of CONF: {{(pid=53971) wait /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/service.py:653}} [ 394.764763] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ******************************************************************************** {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2600}} [ 394.764763] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] Configuration options gathered from: {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2601}} [ 394.764763] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] command line args: ['--config-file', '/etc/nova/nova.conf'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2602}} [ 394.764763] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] config files: ['/etc/nova/nova.conf'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2603}} [ 394.764763] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ================================================================================ {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2605}} [ 394.764763] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] allow_resize_to_same_host = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.765098] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] arq_binding_timeout = 300 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.765144] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] block_device_allocate_retries = 60 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.765306] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] block_device_allocate_retries_interval = 3 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.765517] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cert = self.pem {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.765701] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] compute_driver = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.765935] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] compute_monitors = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.766525] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] config_dir = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.766731] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] config_drive_format = iso9660 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.766876] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] config_file = ['/etc/nova/nova.conf'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.767068] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] config_source = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.767256] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] console_host = devstack {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.767449] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] control_exchange = nova {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.767640] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cpu_allocation_ratio = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.767834] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] daemon = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.768208] nova-conductor[54394]: INFO nova.service [-] Starting conductor node (version 0.0.1) [ 394.768625] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] debug = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.768827] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] default_access_ip_network_name = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.769013] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] default_availability_zone = nova {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.769340] nova-conductor[54393]: INFO dbcounter [None req-8f68fc6b-3f02-48b9-99c6-24cdb00b0cc7 None None] Registered counter for database nova_cell1 [ 394.769483] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] default_ephemeral_format = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.769685] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] default_green_pool_size = 1000 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.770089] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] default_log_levels = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN', 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO', 'oslo.messaging=INFO', 'oslo_messaging=INFO', 'iso8601=WARN', 'requests.packages.urllib3.connectionpool=WARN', 'urllib3.connectionpool=WARN', 'websocket=WARN', 'requests.packages.urllib3.util.retry=WARN', 'urllib3.util.retry=WARN', 'keystonemiddleware=WARN', 'routes.middleware=WARN', 'stevedore=WARN', 'taskflow=WARN', 'keystoneauth=WARN', 'oslo.cache=INFO', 'oslo_policy=INFO', 'dogpile.core.dogpile=INFO', 'glanceclient=WARN', 'oslo.privsep.daemon=INFO'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.770193] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] default_schedule_zone = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.770304] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] disk_allocation_ratio = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.770491] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] enable_new_services = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.770718] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] enabled_apis = ['osapi_compute'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.770891] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] enabled_ssl_apis = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.771074] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] flat_injected = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.771255] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] force_config_drive = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.771434] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] force_raw_images = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.771632] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] graceful_shutdown_timeout = 5 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.771788] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] heal_instance_info_cache_interval = 60 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.772499] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] host = devstack {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.772709] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] initial_cpu_allocation_ratio = 4.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.772890] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] initial_disk_allocation_ratio = 1.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.773060] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] initial_ram_allocation_ratio = 1.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.773330] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] injected_network_template = /opt/stack/nova/nova/virt/interfaces.template {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.773511] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] instance_build_timeout = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.773684] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] instance_delete_interval = 300 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.773888] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] instance_format = [instance: %(uuid)s] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.774016] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] instance_name_template = instance-%08x {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.774169] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] instance_usage_audit = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.774352] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] instance_usage_audit_period = month {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.774522] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] instance_uuid_format = [instance: %(uuid)s] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.774699] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] instances_path = /opt/stack/data/nova/instances {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.774857] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] internal_service_availability_zone = internal {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.775031] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] key = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.775181] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] live_migration_retry_count = 30 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.775406] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] log_color = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.775536] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] log_config_append = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.775711] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] log_date_format = %Y-%m-%d %H:%M:%S {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.776249] nova-conductor[54393]: DEBUG oslo_db.sqlalchemy.engines [None req-8f68fc6b-3f02-48b9-99c6-24cdb00b0cc7 None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=54393) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 394.776317] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] log_dir = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.776317] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] log_file = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.776415] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] log_options = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.776587] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] log_rotate_interval = 1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.776783] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] log_rotate_interval_type = days {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.776945] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] log_rotation_type = none {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.777074] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] logging_context_format_string = %(color)s%(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(project_name)s %(user_name)s%(color)s] %(instance)s%(color)s%(message)s {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.777198] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] logging_debug_format_suffix = {{(pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d}} {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.777368] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] logging_default_format_string = %(color)s%(levelname)s %(name)s [-%(color)s] %(instance)s%(color)s%(message)s {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.777539] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] logging_exception_prefix = ERROR %(name)s %(instance)s {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.777666] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.778009] nova-conductor[54394]: INFO dbcounter [None req-bc0ef412-ba9f-40c3-93c6-4607c36beeef None None] Registered counter for database nova_cell1 [ 394.778245] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] long_rpc_timeout = 1800 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.778441] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] max_concurrent_builds = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.778623] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] max_concurrent_live_migrations = 1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.778777] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] max_concurrent_snapshots = 5 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.778929] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] max_local_block_devices = 3 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.779105] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] max_logfile_count = 30 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.779609] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] max_logfile_size_mb = 200 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.779609] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] maximum_instance_delete_attempts = 5 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.779609] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] metadata_listen = 0.0.0.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.779810] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] metadata_listen_port = 8775 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.779976] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] metadata_workers = 2 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.780150] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] migrate_max_retries = -1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.780309] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] mkisofs_cmd = genisoimage {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.780536] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] my_block_storage_ip = 10.180.1.21 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.780663] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] my_ip = 10.180.1.21 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.780823] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] network_allocate_retries = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.780998] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] non_inheritable_image_properties = ['cache_in_nova', 'bittorrent'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.781165] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] osapi_compute_listen = 0.0.0.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.781331] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] osapi_compute_listen_port = 8774 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.781500] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] osapi_compute_unique_server_name_scope = {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.781660] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] osapi_compute_workers = 2 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.781810] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] password_length = 12 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.781979] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] periodic_enable = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.782161] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] periodic_fuzzy_delay = 60 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.782328] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] pointer_model = usbtablet {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.782511] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] preallocate_images = none {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.782681] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] publish_errors = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.782803] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] pybasedir = /opt/stack/nova {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.782951] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ram_allocation_ratio = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.783118] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] rate_limit_burst = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.783277] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] rate_limit_except_level = CRITICAL {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.783429] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] rate_limit_interval = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.783587] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] reboot_timeout = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.783759] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] reclaim_instance_interval = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.783923] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] record = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.784088] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] reimage_timeout_per_gb = 20 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.784249] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] report_interval = 120 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.784407] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] rescue_timeout = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.784551] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] reserved_host_cpus = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.784730] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] reserved_host_disk_mb = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.784882] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] reserved_host_memory_mb = 512 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.785053] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] reserved_huge_pages = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.785237] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] resize_confirm_window = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.785404] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] resize_fs_using_block_device = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.785558] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] resume_guests_state_on_host_boot = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.785715] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] rootwrap_config = /etc/nova/rootwrap.conf {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.785903] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] rpc_response_timeout = 60 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.786052] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] run_external_periodic_tasks = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.786203] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] running_deleted_instance_action = reap {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.786353] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] running_deleted_instance_poll_interval = 1800 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.786518] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] running_deleted_instance_timeout = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.786667] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] scheduler_instance_sync_interval = 120 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.786824] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] service_down_time = 720 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.787021] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] servicegroup_driver = db {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.787176] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] shelved_offload_time = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.787364] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writer thread running {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 394.787517] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] shelved_poll_interval = 3600 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.787718] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] shutdown_timeout = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.787897] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] source_is_ipv6 = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.788096] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ssl_only = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.788267] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] state_path = /opt/stack/data/nova {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.788439] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] sync_power_state_interval = 600 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.788605] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] sync_power_state_pool_size = 1000 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.788921] nova-conductor[54394]: DEBUG oslo_db.sqlalchemy.engines [None req-bc0ef412-ba9f-40c3-93c6-4607c36beeef None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=54394) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 394.789217] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] syslog_log_facility = LOG_USER {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.789390] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] tempdir = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.789550] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] timeout_nbd = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.789710] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] transport_url = **** {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.789889] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] update_resources_interval = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.790044] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] use_cow_images = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.790198] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] use_eventlog = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.790346] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] use_journal = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.790503] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] use_json = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.790660] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] use_rootwrap_daemon = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.790816] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] use_stderr = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.790968] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] use_syslog = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.791127] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vcpu_pin_set = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.791296] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vif_plugging_is_fatal = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.791471] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vif_plugging_timeout = 300 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.791982] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] virt_mkfs = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.791982] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] volume_usage_poll_interval = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.791982] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] watch_log_file = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.792160] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] web = /usr/share/spice-html5 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 394.792445] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_concurrency.disable_process_locking = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.792664] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_concurrency.lock_path = /opt/stack/data/nova {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.792838] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_metrics.metrics_buffer_size = 1000 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.793001] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_metrics.metrics_enabled = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.793163] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_metrics.metrics_process_name = {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.793325] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_metrics.metrics_socket_file = /var/tmp/metrics_collector.sock {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.793485] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_metrics.metrics_thread_stop_timeout = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.793710] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.auth_strategy = keystone {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.793889] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.compute_link_prefix = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.794072] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.config_drive_skip_versions = 1.0 2007-01-19 2007-03-01 2007-08-29 2007-10-10 2007-12-15 2008-02-01 2008-09-01 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.794256] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.dhcp_domain = novalocal {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.794424] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.enable_instance_password = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.794590] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.glance_link_prefix = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.794748] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.instance_list_cells_batch_fixed_size = 100 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.794927] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.instance_list_cells_batch_strategy = distributed {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.795083] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.instance_list_per_project_cells = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.795233] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.list_records_by_skipping_down_cells = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.795391] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.local_metadata_per_cell = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.795567] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.max_limit = 1000 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.795728] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.metadata_cache_expiration = 15 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.795890] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.neutron_default_tenant_id = default {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.796083] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.response_validation = warn {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.796255] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.use_neutron_default_nets = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.796410] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.vendordata_dynamic_connect_timeout = 5 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.796581] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.vendordata_dynamic_failure_fatal = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.796736] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.vendordata_dynamic_read_timeout = 5 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.796899] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.vendordata_dynamic_ssl_certfile = {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.797063] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.vendordata_dynamic_targets = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.797224] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.vendordata_jsonfile_path = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.797441] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api.vendordata_providers = ['StaticJSON'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.798156] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.backend = dogpile.cache.memcached {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.798156] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.backend_argument = **** {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.798156] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.config_prefix = cache.oslo {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.798341] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.dead_timeout = 60.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.798520] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.debug_cache_backend = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.798686] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.enable_retry_client = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.798862] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.enable_socket_keepalive = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.799026] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.enabled = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.799183] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.enforce_fips_mode = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.799363] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.expiration_time = 600 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.799521] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.hashclient_retry_attempts = 2 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.799717] nova-conductor[54393]: INFO nova.healthcheck.manager [None req-8f68fc6b-3f02-48b9-99c6-24cdb00b0cc7 None None] healthcheck endpoint is not enabled [ 394.800087] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.hashclient_retry_delay = 1.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.800134] nova-conductor[54393]: DEBUG nova.service [None req-8f68fc6b-3f02-48b9-99c6-24cdb00b0cc7 None None] Creating RPC server for service conductor {{(pid=54393) start /opt/stack/nova/nova/service.py:186}} [ 394.800168] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.memcache_dead_retry = 300 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.800334] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.memcache_password = **** {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.800499] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.memcache_pool_connection_get_timeout = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.800659] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.memcache_pool_flush_on_reconnect = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.800850] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.memcache_pool_maxsize = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.801013] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.memcache_pool_unused_timeout = 60 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.801168] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.memcache_sasl_enabled = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.801343] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.memcache_servers = ['localhost:11211'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.801522] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.memcache_socket_timeout = 1.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.801682] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.memcache_username = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.802176] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.proxies = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.802176] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.redis_db = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.802176] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.redis_password = **** {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.802308] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.redis_sentinel_service_name = mymaster {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.802485] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.redis_sentinels = ['localhost:26379'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.802651] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.redis_server = localhost:6379 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.802809] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.redis_socket_timeout = 1.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.802956] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.redis_username = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.803112] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.retry_attempts = 2 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.803289] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.retry_delay = 0.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.803466] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.socket_keepalive_count = 1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.803623] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.socket_keepalive_idle = 1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.803775] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.socket_keepalive_interval = 1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.803924] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.tls_allowed_ciphers = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.804086] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.tls_cafile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.804241] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.tls_certfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.804387] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.tls_enabled = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.804539] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cache.tls_keyfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.804745] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cinder.auth_section = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.804930] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cinder.auth_type = password {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.805085] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cinder.cafile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.805272] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cinder.catalog_info = volumev3::publicURL {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.805425] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cinder.certfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.805591] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cinder.collect_timing = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.805747] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cinder.cross_az_attach = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.805903] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cinder.debug = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.806075] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cinder.endpoint_template = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.806245] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cinder.http_retries = 3 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.806410] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cinder.insecure = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.806586] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cinder.keyfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.806782] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cinder.os_region_name = RegionOne {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.806944] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cinder.split_loggers = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.807100] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cinder.timeout = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.807287] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] compute.consecutive_build_service_disable_threshold = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.807459] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] compute.cpu_dedicated_set = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.807689] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] compute.cpu_shared_set = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.807878] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] compute.image_type_exclude_list = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.808055] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] compute.live_migration_wait_for_vif_plug = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.808216] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] compute.max_concurrent_disk_ops = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.808373] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] compute.max_disk_devices_to_attach = -1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.808549] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] compute.packing_host_numa_cells_allocation_strategy = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.808728] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] compute.provider_config_location = /etc/nova/provider_config/ {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.808908] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] compute.resource_provider_association_refresh = 300 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.809066] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] compute.sharing_providers_max_uuids_per_request = 200 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.809220] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] compute.shutdown_retry_interval = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.809396] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] compute.vmdk_allowed_types = ['streamOptimized', 'monolithicSparse'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.809586] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] conductor.workers = 2 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.809760] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] console.allowed_origins = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.809918] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] console.ssl_ciphers = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.810370] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] console.ssl_minimum_version = default {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.810370] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] consoleauth.enforce_session_timeout = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.810458] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] consoleauth.token_ttl = 600 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.810591] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.cafile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.810806] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.certfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.811004] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.collect_timing = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.811158] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.connect_retries = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.811336] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.connect_retry_delay = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.811529] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.endpoint_override = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.811665] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.insecure = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.812066] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.keyfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.812066] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.max_version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.812155] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.min_version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.812295] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.region_name = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.812445] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.retriable_status_codes = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.812603] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.service_name = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.813051] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.service_type = accelerator {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.813051] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.split_loggers = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.813187] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.status_code_retries = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.813592] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.status_code_retry_delay = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.813592] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.timeout = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.813700] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.valid_interfaces = ['internal', 'public'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.814523] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] cyborg.version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.814523] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.backend = sqlalchemy {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.814523] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.connection = **** {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.814523] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.connection_debug = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.814523] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.connection_parameters = {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.814694] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.connection_recycle_time = 3600 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.814810] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.connection_trace = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.814972] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.db_inc_retry_interval = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.815133] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.db_max_retries = 20 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.815292] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.db_max_retry_interval = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.815543] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.db_retry_interval = 1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.815608] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.max_overflow = 50 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.816458] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.max_pool_size = 5 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.816458] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.max_retries = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.816458] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.mysql_sql_mode = TRADITIONAL {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.816458] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.mysql_wsrep_sync_wait = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.816458] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.pool_timeout = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.816652] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.retry_interval = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.816738] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.slave_connection = **** {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.816883] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.sqlite_synchronous = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.817040] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] database.use_db_reconnect = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.817210] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api_database.backend = sqlalchemy {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.817371] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api_database.connection = **** {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.818030] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api_database.connection_debug = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.818030] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api_database.connection_parameters = {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.818030] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api_database.connection_recycle_time = 3600 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.818163] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api_database.connection_trace = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.818236] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api_database.db_inc_retry_interval = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.818476] nova-conductor[54394]: INFO nova.healthcheck.manager [None req-bc0ef412-ba9f-40c3-93c6-4607c36beeef None None] healthcheck endpoint is not enabled [ 394.819132] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api_database.db_max_retries = 20 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.819206] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writer thread running {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 394.819246] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api_database.db_max_retry_interval = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.819328] nova-conductor[54394]: DEBUG nova.service [None req-bc0ef412-ba9f-40c3-93c6-4607c36beeef None None] Creating RPC server for service conductor {{(pid=54394) start /opt/stack/nova/nova/service.py:186}} [ 394.819359] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api_database.db_retry_interval = 1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.819388] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api_database.max_overflow = 50 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.819548] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api_database.max_pool_size = 5 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.819742] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api_database.max_retries = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.819933] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api_database.mysql_sql_mode = TRADITIONAL {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.820556] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api_database.mysql_wsrep_sync_wait = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.820556] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api_database.pool_timeout = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.820556] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api_database.retry_interval = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.820556] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api_database.slave_connection = **** {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.820704] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] api_database.sqlite_synchronous = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.820996] nova-conductor[54393]: INFO oslo.messaging._drivers.impl_rabbit [None req-8f68fc6b-3f02-48b9-99c6-24cdb00b0cc7 None None] Creating fanout queue: conductor_fanout_5511c5aa7a344d829c6e0dc06ee7bd56 [ 394.821142] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] devices.enabled_mdev_types = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.821484] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ephemeral_storage_encryption.cipher = aes-xts-plain64 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.821547] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ephemeral_storage_encryption.default_format = luks {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.822183] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ephemeral_storage_encryption.enabled = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.822183] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ephemeral_storage_encryption.key_size = 512 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.822183] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.api_servers = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.822183] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.cafile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.822371] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.certfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.822503] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.collect_timing = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.822660] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.connect_retries = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.822821] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.connect_retry_delay = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.822983] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.debug = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.823434] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.default_trusted_certificate_ids = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.823434] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.enable_certificate_validation = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.823434] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.enable_rbd_download = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.823618] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.endpoint_override = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.823782] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.insecure = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.824054] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.keyfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.824214] nova-conductor[54393]: DEBUG nova.service [None req-8f68fc6b-3f02-48b9-99c6-24cdb00b0cc7 None None] Join ServiceGroup membership for this service conductor {{(pid=54393) start /opt/stack/nova/nova/service.py:203}} [ 394.824357] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.max_version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.824556] nova-conductor[54393]: DEBUG nova.servicegroup.drivers.db [None req-8f68fc6b-3f02-48b9-99c6-24cdb00b0cc7 None None] DB_Driver: join new ServiceGroup member devstack to the conductor group, service = {{(pid=54393) join /opt/stack/nova/nova/servicegroup/drivers/db.py:44}} [ 394.824691] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.min_version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.825000] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.num_retries = 3 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.825058] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.rbd_ceph_conf = {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.825222] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.rbd_connect_timeout = 5 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.825370] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.rbd_pool = {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.825539] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.rbd_user = {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.825704] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.region_name = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.825864] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.retriable_status_codes = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.826020] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.service_name = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.826189] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.service_type = image {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.826351] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.split_loggers = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.826600] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.status_code_retries = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.826699] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.status_code_retry_delay = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.826866] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.timeout = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.827047] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.valid_interfaces = ['internal', 'public'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.827212] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.verify_glance_signatures = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.827371] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] glance.version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.827586] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] guestfs.debug = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.827787] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] healthcheck.cache_control = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.827957] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] healthcheck.debug = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.828139] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] healthcheck.ttl = 300 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.828303] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] healthcheck.uri = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.828493] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] mks.enabled = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.829087] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] mks.mksproxy_base_url = http://127.0.0.1:6090/ {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.829302] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] image_cache.manager_interval = 2400 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.829480] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] image_cache.precache_concurrency = 1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.829655] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] image_cache.remove_unused_base_images = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.829833] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] image_cache.remove_unused_original_minimum_age_seconds = 86400 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.830004] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] image_cache.remove_unused_resized_minimum_age_seconds = 3600 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.830182] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] image_cache.subdirectory_name = _base {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.830384] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.api_max_retries = 60 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.830551] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.api_retry_interval = 2 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.830711] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.auth_section = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.830873] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.auth_type = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.831030] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.cafile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.831185] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.certfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.831348] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.collect_timing = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.831530] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.conductor_group = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.831718] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.connect_retries = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.831907] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.connect_retry_delay = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.832104] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.endpoint_override = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.832255] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.insecure = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.832413] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.keyfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.832571] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.max_version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.832737] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.min_version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.832906] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.peer_list = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.833062] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.region_name = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.833216] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.retriable_status_codes = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.833377] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.serial_console_state_timeout = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.833535] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.service_name = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.833726] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.service_type = baremetal {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.833892] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.shard = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.834055] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.split_loggers = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.834215] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.status_code_retries = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.834372] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.status_code_retry_delay = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.834531] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.timeout = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.834714] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.valid_interfaces = ['internal', 'public'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.834876] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ironic.version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.835083] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] key_manager.backend = nova.keymgr.conf_key_mgr.ConfKeyManager {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.835716] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] key_manager.fixed_key = **** {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.835716] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican.auth_endpoint = http://localhost/identity/v3 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.835716] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican.barbican_api_version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.835842] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican.barbican_endpoint = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.836062] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican.barbican_endpoint_type = public {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.836249] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican.barbican_region_name = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.836426] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican.cafile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.836733] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican.certfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.836822] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican.collect_timing = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.837031] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican.insecure = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.837169] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican.keyfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.837654] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican.number_of_retries = 60 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.837654] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican.retry_delay = 1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.837806] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican.send_service_user_token = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.837854] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican.split_loggers = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.838248] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican.timeout = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.838248] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican.verify_ssl = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.838387] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican.verify_ssl_path = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.838540] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican_service_user.auth_section = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.838705] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican_service_user.auth_type = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.838862] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican_service_user.cafile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.839018] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican_service_user.certfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.839179] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican_service_user.collect_timing = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.839361] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican_service_user.insecure = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.839488] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican_service_user.keyfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.839677] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican_service_user.split_loggers = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.840090] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] barbican_service_user.timeout = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.840090] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vault.approle_role_id = **** {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.840242] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vault.approle_secret_id = **** {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.840315] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vault.kv_mountpoint = secret {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.840473] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vault.kv_path = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.840714] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vault.kv_version = 2 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.840794] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vault.namespace = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.841131] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vault.root_token_id = **** {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.841131] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vault.ssl_ca_crt_file = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.841780] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vault.timeout = 60.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.841834] nova-conductor[54394]: INFO oslo.messaging._drivers.impl_rabbit [None req-bc0ef412-ba9f-40c3-93c6-4607c36beeef None None] Creating fanout queue: conductor_fanout_3c4e27c8268e4fe6a1bd520578579cda [ 394.841885] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vault.use_ssl = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.841885] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vault.vault_url = http://127.0.0.1:8200 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.842111] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.auth_section = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.842229] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.auth_type = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.842814] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.cafile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.842814] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.certfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.842814] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.collect_timing = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.843012] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.connect_retries = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.843012] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.connect_retry_delay = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.843188] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.endpoint_override = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.843310] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.insecure = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.843458] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.keyfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.843661] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.max_version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.843784] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.min_version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.844133] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.region_name = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.844235] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.retriable_status_codes = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.844564] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.service_name = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.844564] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.service_type = identity {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.844705] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.split_loggers = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.844818] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.status_code_retries = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.844972] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.status_code_retry_delay = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.845170] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.timeout = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.845364] nova-conductor[54394]: DEBUG nova.service [None req-bc0ef412-ba9f-40c3-93c6-4607c36beeef None None] Join ServiceGroup membership for this service conductor {{(pid=54394) start /opt/stack/nova/nova/service.py:203}} [ 394.845509] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.valid_interfaces = ['internal', 'public'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.845778] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] keystone.version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.845877] nova-conductor[54394]: DEBUG nova.servicegroup.drivers.db [None req-bc0ef412-ba9f-40c3-93c6-4607c36beeef None None] DB_Driver: join new ServiceGroup member devstack to the conductor group, service = {{(pid=54394) join /opt/stack/nova/nova/servicegroup/drivers/db.py:44}} [ 394.846024] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.connection_uri = {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.846224] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.cpu_mode = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.846523] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.cpu_model_extra_flags = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.846705] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.cpu_models = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.846930] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.cpu_power_governor_high = performance {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.847102] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.cpu_power_governor_low = powersave {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.847269] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.cpu_power_management = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.847480] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.cpu_power_management_strategy = cpu_state {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.847731] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.device_detach_attempts = 8 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.847904] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.device_detach_timeout = 20 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.848088] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.disk_cachemodes = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.848252] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.disk_prefix = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.848417] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.enabled_perf_events = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.848577] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.file_backed_memory = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.848741] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.gid_maps = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.848899] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.hw_disk_discard = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.849060] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.hw_machine_type = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.849229] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.images_rbd_ceph_conf = {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.849389] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.images_rbd_glance_copy_poll_interval = 15 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.849549] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.images_rbd_glance_copy_timeout = 600 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.849723] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.images_rbd_glance_store_name = {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.849887] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.images_rbd_pool = rbd {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.850083] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.images_type = default {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.850208] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.images_volume_group = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.850372] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.inject_key = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.850532] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.inject_partition = -2 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.850720] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.inject_password = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.850933] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.iscsi_iface = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.851120] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.iser_use_multipath = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.851290] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.live_migration_bandwidth = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.851451] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.live_migration_completion_timeout = 800 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.851616] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.live_migration_downtime = 500 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.851777] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.live_migration_downtime_delay = 75 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.851935] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.live_migration_downtime_steps = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.852109] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.live_migration_inbound_addr = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.852272] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.live_migration_permit_auto_converge = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.852434] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.live_migration_permit_post_copy = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.852595] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.live_migration_scheme = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.852817] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.live_migration_timeout_action = abort {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.852984] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.live_migration_tunnelled = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.853144] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.live_migration_uri = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.853308] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.live_migration_with_native_tls = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.853506] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.max_queues = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.853691] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.mem_stats_period_seconds = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.853939] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.migration_inbound_addr = 10.180.1.21 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.854140] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.nfs_mount_options = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.854373] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.nfs_mount_point_base = /opt/stack/data/nova/mnt {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.854538] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.num_aoe_discover_tries = 3 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.854701] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.num_iser_scan_tries = 5 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.854859] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.num_memory_encrypted_guests = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.855020] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.num_nvme_discover_tries = 5 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.855180] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.num_pcie_ports = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.855341] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.num_volume_scan_tries = 5 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.855511] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.pmem_namespaces = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.855668] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.quobyte_client_cfg = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.856378] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.quobyte_mount_point_base = /opt/stack/data/nova/mnt {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.856578] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.rbd_connect_timeout = 5 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.856752] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.rbd_destroy_volume_retries = 12 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.856992] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.rbd_destroy_volume_retry_interval = 5 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.857069] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.rbd_secret_uuid = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.857229] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.rbd_user = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.857391] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.realtime_scheduler_priority = 1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.857568] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.remote_filesystem_transport = ssh {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.857751] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.rescue_image_id = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.857908] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.rescue_kernel_id = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.858063] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.rescue_ramdisk_id = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.858228] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.rng_dev_path = /dev/urandom {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.858385] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.rx_queue_size = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.858552] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.smbfs_mount_options = {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.858772] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.smbfs_mount_point_base = /opt/stack/data/nova/mnt {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.858944] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.snapshot_compression = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.859102] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.snapshot_image_format = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.859314] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.snapshots_directory = /opt/stack/data/nova/instances/snapshots {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.859479] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.sparse_logical_volumes = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.859642] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.swtpm_enabled = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.859829] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.swtpm_group = tss {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.859996] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.swtpm_user = tss {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.860181] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.sysinfo_serial = unique {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.860341] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.tb_cache_size = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.860504] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.tx_queue_size = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.860671] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.uid_maps = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.860835] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.use_virtio_for_bridges = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.861001] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.virt_type = kvm {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.861172] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.volume_clear = zero {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.861333] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.volume_clear_size = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.861518] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.volume_use_multipath = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.861683] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.vzstorage_cache_path = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.861869] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.vzstorage_log_path = /var/log/vstorage/%(cluster_name)s/nova.log.gz {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.862091] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.vzstorage_mount_group = qemu {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.862265] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.vzstorage_mount_opts = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.862438] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.vzstorage_mount_perms = 0770 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.862653] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.vzstorage_mount_point_base = /opt/stack/data/nova/mnt {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.862832] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.vzstorage_mount_user = stack {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.862994] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] libvirt.wait_soft_reboot_seconds = 120 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.863190] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.auth_section = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.863360] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.auth_type = password {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.863523] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.cafile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.863683] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.certfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.863846] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.collect_timing = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.864053] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.connect_retries = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.864266] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.connect_retry_delay = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.864447] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.default_floating_pool = public {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.864608] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.endpoint_override = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.864774] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.extension_sync_interval = 600 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.864936] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.http_retries = 3 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.865097] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.insecure = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.865255] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.keyfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.865413] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.max_version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.865581] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.metadata_proxy_shared_secret = **** {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.865739] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.min_version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.865904] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.ovs_bridge = br-int {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.866115] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.physnets = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.866284] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.region_name = RegionOne {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.866468] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.retriable_status_codes = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.866658] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.service_metadata_proxy = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.866821] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.service_name = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.866992] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.service_type = network {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.867174] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.split_loggers = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.867332] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.status_code_retries = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.867488] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.status_code_retry_delay = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.867690] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.timeout = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.867898] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.valid_interfaces = ['internal', 'public'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.868403] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] neutron.version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.868403] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] notifications.bdms_in_notifications = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.868538] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] notifications.default_level = INFO {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.868593] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] notifications.notification_format = unversioned {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.868775] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] notifications.notify_on_state_change = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.868955] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] notifications.versioned_notifications_topics = ['versioned_notifications'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.869136] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] pci.alias = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.869308] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] pci.device_spec = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.869473] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] pci.report_in_placement = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.869678] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.auth_section = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.869867] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.auth_type = password {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.870054] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.auth_url = http://10.180.1.21/identity {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.870217] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.cafile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.870373] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.certfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.870912] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.collect_timing = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.870912] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.connect_retries = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.870912] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.connect_retry_delay = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.871065] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.default_domain_id = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.871146] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.default_domain_name = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.871301] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.domain_id = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.871455] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.domain_name = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.871614] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.endpoint_override = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.871774] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.insecure = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.871933] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.keyfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.872102] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.max_version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.872262] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.min_version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.872432] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.password = **** {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.872592] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.project_domain_id = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.872757] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.project_domain_name = Default {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.872926] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.project_id = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.873096] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.project_name = service {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.873259] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.region_name = RegionOne {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.873418] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.retriable_status_codes = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.873573] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.service_name = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.873738] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.service_type = placement {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.873898] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.split_loggers = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.874055] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.status_code_retries = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.874215] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.status_code_retry_delay = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.874372] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.system_scope = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.874524] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.timeout = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.874686] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.trust_id = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.874840] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.user_domain_id = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.875002] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.user_domain_name = Default {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.875186] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.user_id = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.875368] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.username = nova {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.875545] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.valid_interfaces = ['internal', 'public'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.875705] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] placement.version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.875884] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] quota.cores = 20 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.876063] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] quota.count_usage_from_placement = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.876236] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] quota.driver = nova.quota.DbQuotaDriver {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.876427] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] quota.injected_file_content_bytes = 10240 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.876618] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] quota.injected_file_path_length = 255 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.876790] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] quota.injected_files = 5 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.876960] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] quota.instances = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.877124] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] quota.key_pairs = 100 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.877293] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] quota.metadata_items = 128 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.877459] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] quota.ram = 51200 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.877655] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] quota.recheck_quota = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.877841] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] quota.server_group_members = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.878009] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] quota.server_groups = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.878229] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] scheduler.discover_hosts_in_cells_interval = -1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.878419] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] scheduler.enable_isolated_aggregate_filtering = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.878585] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] scheduler.image_metadata_prefilter = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.878769] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] scheduler.limit_tenants_to_placement_aggregate = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.878931] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] scheduler.max_attempts = 3 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.879092] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] scheduler.max_placement_results = 1000 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.879273] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] scheduler.placement_aggregate_required_for_tenants = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.879438] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] scheduler.query_placement_for_image_type_support = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.879615] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] scheduler.query_placement_for_routed_network_aggregates = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.879835] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] scheduler.workers = 2 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.880136] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.aggregate_image_properties_isolation_namespace = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.880325] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.aggregate_image_properties_isolation_separator = . {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.880529] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.available_filters = ['nova.scheduler.filters.all_filters'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.880704] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.build_failure_weight_multiplier = 1000000.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.880869] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.cpu_weight_multiplier = 1.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.881033] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.cross_cell_move_weight_multiplier = 1000000.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.881195] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.disk_weight_multiplier = 1.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.881408] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.enabled_filters = ['ComputeFilter', 'ComputeCapabilitiesFilter', 'ImagePropertiesFilter', 'ServerGroupAntiAffinityFilter', 'ServerGroupAffinityFilter', 'SameHostFilter', 'DifferentHostFilter'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.881579] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.host_subset_size = 1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.881748] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.hypervisor_version_weight_multiplier = 1.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.881926] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.image_properties_default_architecture = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.882089] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.io_ops_weight_multiplier = -1.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.882254] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.isolated_hosts = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.882415] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.isolated_images = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.882575] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.max_instances_per_host = 50 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.882742] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.max_io_ops_per_host = 8 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.882904] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.num_instances_weight_multiplier = 0.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.883086] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.pci_in_placement = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.883248] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.pci_weight_multiplier = 1.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.883409] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.ram_weight_multiplier = 1.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.883570] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.restrict_isolated_hosts_to_isolated_images = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.883746] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.shuffle_best_same_weighed_hosts = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.883938] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.soft_affinity_weight_multiplier = 1.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.884120] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.soft_anti_affinity_weight_multiplier = 1.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.884309] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.track_instance_changes = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.884488] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] filter_scheduler.weight_classes = ['nova.scheduler.weights.all_weighers'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.884661] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] metrics.required = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.884829] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] metrics.weight_multiplier = 1.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.884998] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] metrics.weight_of_unavailable = -10000.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.885164] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] metrics.weight_setting = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.885485] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] serial_console.base_url = ws://127.0.0.1:6083/ {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.885663] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] serial_console.enabled = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.885855] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] serial_console.port_range = 10000:20000 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.886025] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] serial_console.proxyclient_address = 127.0.0.1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.886191] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] serial_console.serialproxy_host = 0.0.0.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.886355] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] serial_console.serialproxy_port = 6083 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.886542] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] service_user.auth_section = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.886724] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] service_user.auth_type = password {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.886883] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] service_user.cafile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.887040] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] service_user.certfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.887199] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] service_user.collect_timing = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.887356] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] service_user.insecure = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.887511] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] service_user.keyfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.887705] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] service_user.send_service_user_token = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.887874] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] service_user.split_loggers = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.888044] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] service_user.timeout = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.888254] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] spice.agent_enabled = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.888441] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] spice.enabled = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.888782] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] spice.html5proxy_base_url = http://127.0.0.1:6082/spice_auto.html {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.888991] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] spice.html5proxy_host = 0.0.0.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.889183] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] spice.html5proxy_port = 6082 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.889343] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] spice.image_compression = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.889501] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] spice.jpeg_compression = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.889665] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] spice.playback_compression = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.889825] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] spice.server_listen = 127.0.0.1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.889988] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] spice.server_proxyclient_address = 127.0.0.1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.890144] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] spice.streaming_mode = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.890296] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] spice.zlib_compression = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.890487] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] upgrade_levels.baseapi = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.890656] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] upgrade_levels.compute = auto {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.890811] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] upgrade_levels.conductor = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.890965] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] upgrade_levels.scheduler = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.891127] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vendordata_dynamic_auth.auth_section = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.891291] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vendordata_dynamic_auth.auth_type = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.891445] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vendordata_dynamic_auth.cafile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.891598] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vendordata_dynamic_auth.certfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.891786] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vendordata_dynamic_auth.collect_timing = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.891962] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vendordata_dynamic_auth.insecure = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.892134] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vendordata_dynamic_auth.keyfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.892324] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vendordata_dynamic_auth.split_loggers = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.892483] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vendordata_dynamic_auth.timeout = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.892684] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.api_retry_count = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.892842] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.ca_file = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.893027] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.cache_prefix = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.893184] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.cluster_name = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.893341] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.connection_pool_size = 10 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.893514] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.console_delay_seconds = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.893675] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.datastore_regex = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.893832] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.host_ip = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.893987] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.host_password = **** {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.894147] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.host_port = 443 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.894301] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.host_username = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.894459] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.insecure = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.894612] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.integration_bridge = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.894775] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.maximum_objects = 100 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.894928] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.pbm_default_policy = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.895085] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.pbm_enabled = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.895238] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.pbm_wsdl_location = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.895407] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.serial_log_dir = /opt/vmware/vspc {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.895564] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.serial_port_proxy_uri = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.895720] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.serial_port_service_uri = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.895883] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.task_poll_interval = 0.5 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.896067] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.use_linked_clone = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.896221] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.vnc_keymap = en-us {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.896388] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.vnc_port = 5900 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.896578] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vmware.vnc_port_total = 10000 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.896786] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vnc.auth_schemes = ['none'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.896956] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vnc.enabled = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.897268] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vnc.novncproxy_base_url = http://127.0.0.1:6080/vnc_auto.html {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.897457] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vnc.novncproxy_host = 0.0.0.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.897643] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vnc.novncproxy_port = 6080 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.897837] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vnc.server_listen = 127.0.0.1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.898011] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vnc.server_proxyclient_address = 127.0.0.1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.898176] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vnc.vencrypt_ca_certs = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.898333] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vnc.vencrypt_client_cert = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.898489] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] vnc.vencrypt_client_key = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.898702] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.disable_compute_service_check_for_ffu = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.898870] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.disable_deep_image_inspection = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.899029] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.disable_fallback_pcpu_query = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.899186] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.disable_group_policy_check_upcall = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.899342] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.disable_libvirt_livesnapshot = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.899500] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.disable_rootwrap = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.899659] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.enable_numa_live_migration = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.899818] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.enable_qemu_monitor_announce_self = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.899976] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.ensure_libvirt_rbd_instance_dir_cleanup = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.900153] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.handle_virt_lifecycle_events = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.900313] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.libvirt_disable_apic = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.900472] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.never_download_image_if_on_rbd = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.900632] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.qemu_monitor_announce_self_count = 3 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.900788] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.qemu_monitor_announce_self_interval = 1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.900975] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.reserve_disk_resource_for_image_cache = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.901130] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.skip_cpu_compare_at_startup = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.901286] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.skip_cpu_compare_on_dest = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.901441] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.skip_hypervisor_version_check_on_lm = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.901598] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.skip_reserve_in_use_ironic_nodes = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.902170] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.unified_limits_count_pcpu_as_vcpu = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.902170] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] workarounds.wait_for_vif_plugged_event_during_hard_reboot = [] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.902317] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] wsgi.api_paste_config = /etc/nova/api-paste.ini {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.902415] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] wsgi.client_socket_timeout = 900 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.902591] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] wsgi.default_pool_size = 1000 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.902763] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] wsgi.keep_alive = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.902930] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] wsgi.max_header_line = 16384 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.903094] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] wsgi.secure_proxy_ssl_header = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.903261] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] wsgi.ssl_ca_file = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.903422] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] wsgi.ssl_cert_file = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.903583] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] wsgi.ssl_key_file = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.903751] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] wsgi.tcp_keepidle = 600 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.903949] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] wsgi.wsgi_log_format = %(client_ip)s "%(request_line)s" status: %(status_code)s len: %(body_length)s time: %(wall_seconds).7f {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.904204] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] zvm.ca_file = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.904295] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] zvm.cloud_connector_url = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.904513] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] zvm.image_tmp_path = /opt/stack/data/nova/images {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.904681] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] zvm.reachable_timeout = 300 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.904913] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_policy.enforce_new_defaults = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.905090] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_policy.enforce_scope = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.905282] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_policy.policy_default_rule = default {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.905466] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_policy.policy_dirs = ['policy.d'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.905641] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_policy.policy_file = policy.yaml {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.905834] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_policy.remote_content_type = application/x-www-form-urlencoded {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.905998] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_policy.remote_ssl_ca_crt_file = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.906154] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_policy.remote_ssl_client_crt_file = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.906309] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_policy.remote_ssl_client_key_file = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.906471] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_policy.remote_ssl_verify_server_crt = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.906715] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_versionedobjects.fatal_exception_format_errors = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.906952] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_middleware.http_basic_auth_user_file = /etc/htpasswd {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.907172] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] profiler.connection_string = messaging:// {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.907346] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] profiler.enabled = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.907553] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] profiler.es_doc_type = notification {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.907736] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] profiler.es_scroll_size = 10000 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.907914] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] profiler.es_scroll_time = 2m {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.908107] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] profiler.filter_error_trace = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.908286] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] profiler.hmac_keys = **** {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.908452] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] profiler.sentinel_service_name = mymaster {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.908616] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] profiler.socket_timeout = 0.1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.908780] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] profiler.trace_requests = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.908939] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] profiler.trace_sqlalchemy = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.909149] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] profiler_jaeger.process_tags = {} {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.909308] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] profiler_jaeger.service_name_prefix = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.909469] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] profiler_otlp.service_name_prefix = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.909673] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] remote_debug.host = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.909854] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] remote_debug.port = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.910046] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.amqp_auto_delete = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.910213] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.amqp_durable_queues = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.910376] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.conn_pool_min_size = 2 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.910536] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.conn_pool_ttl = 1200 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.910702] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.direct_mandatory_flag = True {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.910862] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.enable_cancel_on_failover = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.911025] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.heartbeat_in_pthread = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.911186] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.heartbeat_rate = 3 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.911341] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.heartbeat_timeout_threshold = 60 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.911528] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.hostname = devstack {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.911692] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.kombu_compression = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.911915] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.kombu_failover_strategy = round-robin {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.912122] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.kombu_missing_consumer_retry_timeout = 60 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.912303] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.kombu_reconnect_delay = 1.0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.912474] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.processname = nova-conductor {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.912642] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.rabbit_ha_queues = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.912803] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.rabbit_interval_max = 30 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.912971] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.rabbit_login_method = AMQPLAIN {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.913134] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.rabbit_qos_prefetch_count = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.913290] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.rabbit_quorum_delivery_limit = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.913450] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.rabbit_quorum_max_memory_bytes = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.913608] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.rabbit_quorum_max_memory_length = 0 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.913769] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.rabbit_quorum_queue = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.913929] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.rabbit_retry_backoff = 2 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.914087] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.rabbit_retry_interval = 1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.914282] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.rabbit_stream_fanout = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.914452] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.rabbit_transient_queues_ttl = 1800 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.914640] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.rabbit_transient_quorum_queue = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.914815] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.rpc_conn_pool_size = 30 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.914984] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.ssl = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.915153] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.ssl_ca_file = {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.915318] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.ssl_cert_file = {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.915480] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.ssl_enforce_fips_mode = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.915648] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.ssl_key_file = {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.915820] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.ssl_version = {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.915978] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_rabbit.use_queue_manager = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.916218] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_notifications.driver = ['messagingv2'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.916387] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_notifications.retry = -1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.916615] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_notifications.topics = ['notifications'] {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.916798] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_messaging_notifications.transport_url = **** {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.917000] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.auth_section = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.917181] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.auth_type = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.917342] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.cafile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.917499] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.certfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.917689] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.collect_timing = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.917857] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.connect_retries = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.918015] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.connect_retry_delay = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.918172] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.endpoint_id = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.918328] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.endpoint_override = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.918489] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.insecure = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.918647] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.keyfile = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.918859] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.max_version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.919045] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.min_version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.919208] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.region_name = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.919365] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.retriable_status_codes = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.919525] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.service_name = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.919680] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.service_type = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.919838] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.split_loggers = False {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.919994] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.status_code_retries = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.920167] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.status_code_retry_delay = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.920343] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.timeout = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.920523] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.valid_interfaces = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.920681] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_limit.version = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.920879] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_reports.file_event_handler = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.921067] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_reports.file_event_handler_interval = 1 {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.921223] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] oslo_reports.log_dir = None {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 394.921353] nova-conductor[53971]: DEBUG oslo_service.service [None req-b778c826-1199-4db7-a953-f80fd049c503 None None] ******************************************************************************** {{(pid=53971) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2624}} [ 404.659816] nova-conductor[53971]: DEBUG dbcounter [-] [53971] Writing DB stats nova_api:SELECT=1 {{(pid=53971) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 404.752468] nova-conductor[53971]: DEBUG dbcounter [-] [53971] Writing DB stats nova_cell0:SELECT=1 {{(pid=53971) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 404.753296] nova-conductor[53971]: DEBUG dbcounter [-] [53971] Writing DB stats nova_cell1:SELECT=1 {{(pid=53971) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 409.834836] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=4,nova_cell1:INSERT=1,nova_cell1:UPDATE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 409.854594] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=3 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 481.350145] nova-conductor[54394]: DEBUG oslo_db.sqlalchemy.engines [None req-64f566ea-e7ab-467a-8cf1-99c459f7961f None None] Parent process 53971 forked (54394) with an open database connection, which is being discarded and recreated. {{(pid=54394) checkout /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:438}} [ 481.352495] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writer thread running {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 481.352883] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=2 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 498.506078] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=7,nova_cell1:INSERT=1,nova_cell1:UPDATE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 504.037976] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=7,nova_cell1:INSERT=1,nova_cell1:UPDATE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 523.839821] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writer thread running {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 523.840240] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=2 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 523.849852] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Creating reply queue: reply_65c7891fb9ba47458ad860069679c1e0 [ 523.862098] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Expecting reply to msg db53f6a8ff554a3d8ac72260288a6138 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 524.595607] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg db53f6a8ff554a3d8ac72260288a6138 [ 524.866494] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Took 1.02 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 524.898694] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 524.898971] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 524.901028] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.002s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 524.903027] nova-conductor[54393]: INFO dbcounter [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Registered counter for database nova_cell1 [ 524.906478] nova-conductor[54393]: DEBUG oslo_db.sqlalchemy.engines [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=54393) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 524.911797] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writer thread running {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 524.992333] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 524.992576] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 524.993029] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 524.993467] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 524.993651] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 524.993804] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 524.995122] nova-conductor[54393]: INFO dbcounter [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Registered counter for database nova_cell0 [ 525.003814] nova-conductor[54393]: DEBUG oslo_db.sqlalchemy.engines [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=54393) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 525.008469] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writer thread running {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 525.021495] nova-conductor[54393]: DEBUG nova.quota [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Getting quotas for project b1ee777ca4db465cadf6bb6f0097156d. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 525.025127] nova-conductor[54393]: DEBUG nova.quota [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Getting quotas for user 6d55421997ff49d4ba9834bd871fdff1 and project b1ee777ca4db465cadf6bb6f0097156d. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 525.030597] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] [instance: d20f0dc1-16c6-4d1f-8a50-d3a823343986] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 525.031179] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 525.031379] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 525.031541] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 525.049703] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] [instance: d20f0dc1-16c6-4d1f-8a50-d3a823343986] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 525.049703] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 525.049703] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 525.049989] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 525.107235] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 525.107638] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 525.108817] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 525.108817] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Acquiring lock "compute-rpcapi-router" {{(pid=54393) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 525.108817] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Acquired lock "compute-rpcapi-router" {{(pid=54393) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 525.109127] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2c50aa17-9476-45e0-af68-ef020f4d87ed None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 525.109428] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2c50aa17-9476-45e0-af68-ef020f4d87ed None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 525.109738] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2c50aa17-9476-45e0-af68-ef020f4d87ed None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 525.110207] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2c50aa17-9476-45e0-af68-ef020f4d87ed None None] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 525.110722] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2c50aa17-9476-45e0-af68-ef020f4d87ed None None] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 525.111084] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2c50aa17-9476-45e0-af68-ef020f4d87ed None None] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 525.119178] nova-conductor[54393]: INFO nova.compute.rpcapi [None req-2c50aa17-9476-45e0-af68-ef020f4d87ed None None] Automatically selected compute RPC version 6.3 from minimum service version 67 [ 525.119850] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2c50aa17-9476-45e0-af68-ef020f4d87ed None None] Releasing lock "compute-rpcapi-router" {{(pid=54393) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 526.766569] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Creating reply queue: reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 526.774433] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Expecting reply to msg e03f31745d214ef8b5e5a96c0f140b00 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 527.019944] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e03f31745d214ef8b5e5a96c0f140b00 [ 527.277329] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Took 0.52 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 527.307861] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 527.308379] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 527.310341] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.002s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 527.313102] nova-conductor[54394]: INFO dbcounter [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Registered counter for database nova_cell1 [ 527.325769] nova-conductor[54394]: DEBUG oslo_db.sqlalchemy.engines [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=54394) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 527.342633] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writer thread running {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 527.410879] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 527.411137] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 527.411645] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 527.412220] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 527.412452] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 527.412737] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 527.414245] nova-conductor[54394]: INFO dbcounter [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Registered counter for database nova_cell0 [ 527.434824] nova-conductor[54394]: DEBUG oslo_db.sqlalchemy.engines [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=54394) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 527.444736] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writer thread running {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 527.462716] nova-conductor[54394]: DEBUG nova.quota [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Getting quotas for project df2fd5a65d244a338eb8f63598dac6a0. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 527.467758] nova-conductor[54394]: DEBUG nova.quota [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Getting quotas for user ca4f0f50cbfb4499a8ad482c06431bae and project df2fd5a65d244a338eb8f63598dac6a0. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 527.473480] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] [instance: 1c872581-1c03-4724-9dcd-31cb4c270e5d] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 527.474057] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 527.474257] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 527.474495] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 527.483891] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] [instance: 1c872581-1c03-4724-9dcd-31cb4c270e5d] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 527.484907] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 527.485114] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 527.485278] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 527.524371] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Expecting reply to msg 3eeb36d7d91a4170bb2da16be699b215 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 527.529042] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 527.529042] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 527.529283] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 527.529464] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Acquiring lock "compute-rpcapi-router" {{(pid=54394) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 527.529571] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Acquired lock "compute-rpcapi-router" {{(pid=54394) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 527.530120] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e8661d3-a182-497f-a492-ff6c5535a037 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 527.530303] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e8661d3-a182-497f-a492-ff6c5535a037 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 527.530456] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e8661d3-a182-497f-a492-ff6c5535a037 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 527.530807] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e8661d3-a182-497f-a492-ff6c5535a037 None None] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 527.530966] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e8661d3-a182-497f-a492-ff6c5535a037 None None] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 527.531117] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e8661d3-a182-497f-a492-ff6c5535a037 None None] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 527.540287] nova-conductor[54394]: INFO nova.compute.rpcapi [None req-1e8661d3-a182-497f-a492-ff6c5535a037 None None] Automatically selected compute RPC version 6.3 from minimum service version 67 [ 527.540789] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e8661d3-a182-497f-a492-ff6c5535a037 None None] Releasing lock "compute-rpcapi-router" {{(pid=54394) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 527.715780] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3eeb36d7d91a4170bb2da16be699b215 [ 527.745345] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Expecting reply to msg 28e05dc1601c4ee7a30b8f2cfbcb4dd2 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 527.921991] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 28e05dc1601c4ee7a30b8f2cfbcb4dd2 [ 528.027058] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 528.041498] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 528.041805] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 528.042008] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 528.138966] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 528.154500] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.014s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 528.154837] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.002s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 528.155296] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 528.156317] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 528.156802] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 528.178193] nova-conductor[54394]: DEBUG nova.quota [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Getting quotas for project d454bab6b61047758ec165d8e6e5da20. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 528.180148] nova-conductor[54394]: DEBUG nova.quota [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Getting quotas for user 756f2fbb7787492091ab9f5d544e612f and project d454bab6b61047758ec165d8e6e5da20. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 528.185794] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] [instance: aea5f1ef-c521-4419-9d23-f2f3cc53d13a] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 528.186353] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 528.186570] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 528.186744] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 528.197860] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] [instance: aea5f1ef-c521-4419-9d23-f2f3cc53d13a] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 528.198702] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 528.199124] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 528.199352] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 528.251930] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 528.270694] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 528.270694] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 528.270694] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 528.283999] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 528.284553] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 528.285684] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 528.332090] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 528.332331] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 528.332494] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 528.337767] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 528.337982] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 528.338352] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 528.365232] nova-conductor[54394]: DEBUG nova.quota [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Getting quotas for project 35016e81565b4050bb44cda3400dda95. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 528.374697] nova-conductor[54394]: DEBUG nova.quota [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Getting quotas for user 6611b6fa2bc9413e9252c883ddf8a307 and project 35016e81565b4050bb44cda3400dda95. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 528.393332] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] [instance: ab397815-5da5-4974-8448-b1ffa8d48345] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 528.394003] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 528.394251] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 528.394471] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 528.398241] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] [instance: ab397815-5da5-4974-8448-b1ffa8d48345] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 528.398973] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 528.399210] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 528.399423] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 528.429426] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 528.429426] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 528.429426] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 528.759742] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Expecting reply to msg 73716fe8f5c5429d8228b86e47bee2cc in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 528.941251] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 73716fe8f5c5429d8228b86e47bee2cc [ 529.262370] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Took 0.50 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 529.279458] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 529.281443] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 529.281443] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 529.320088] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 529.320533] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 529.320817] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 529.324521] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 529.324856] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.003s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 529.325286] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 529.341528] nova-conductor[54393]: DEBUG nova.quota [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Getting quotas for project 363c278ffea34f4497b900bd72628403. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 529.343653] nova-conductor[54393]: DEBUG nova.quota [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Getting quotas for user f737f327e72043bca075a6fa801534d8 and project 363c278ffea34f4497b900bd72628403. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 529.349178] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] [instance: 48bf3159-b4ce-4bbf-b4da-02b824326450] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 529.349923] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 529.350226] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 529.350489] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 529.356960] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] [instance: 48bf3159-b4ce-4bbf-b4da-02b824326450] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 529.358056] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 529.358386] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 529.358686] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 529.378343] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 529.378865] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 529.379146] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 530.416731] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Expecting reply to msg 0fa0e5cb43d044ecb6b4091ec4cc4615 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 530.607796] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0fa0e5cb43d044ecb6b4091ec4cc4615 [ 530.919781] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 530.940143] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 530.940143] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 530.940143] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 530.988267] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 530.988569] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 530.988854] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 530.989296] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 530.989517] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 530.989726] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 531.002916] nova-conductor[54394]: DEBUG nova.quota [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Getting quotas for project f141fdb4ee0e4ac3840b1f33375220d1. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 531.005130] nova-conductor[54394]: DEBUG nova.quota [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Getting quotas for user 23991cb0cc0f4926a59342bbc72a6320 and project f141fdb4ee0e4ac3840b1f33375220d1. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 531.010510] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] [instance: 7e688ee8-3f7a-42bc-9e2f-bdd850c2c950] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 531.011173] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 531.011404] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 531.011606] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 531.020766] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] [instance: 7e688ee8-3f7a-42bc-9e2f-bdd850c2c950] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 531.021744] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 531.022258] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 531.023513] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 531.058596] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 531.058921] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 531.059165] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 531.805408] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Expecting reply to msg 8169f92627c944e885f52018ff874580 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 531.991558] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8169f92627c944e885f52018ff874580 [ 532.311441] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 532.342225] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 532.342225] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 532.342225] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 532.382489] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 532.382770] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 532.383176] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 532.383811] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 532.383979] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 532.384310] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 532.397369] nova-conductor[54394]: DEBUG nova.quota [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Getting quotas for project 3109f32864ed4f12909632170a54663f. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 532.400100] nova-conductor[54394]: DEBUG nova.quota [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Getting quotas for user 99547e852bc54a53859de06d58f7a460 and project 3109f32864ed4f12909632170a54663f. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 532.409015] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] [instance: b2fc6a2b-f913-49c4-8339-9ad8e806dbf0] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 532.409597] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 532.409846] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 532.410178] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 532.414913] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] [instance: b2fc6a2b-f913-49c4-8339-9ad8e806dbf0] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 532.415661] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 532.415906] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 532.416172] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 532.434108] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 532.434384] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 532.434593] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 534.909804] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Expecting reply to msg c1c2baf9aac7456f9e0f2ac56379ebed in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 535.072623] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c1c2baf9aac7456f9e0f2ac56379ebed [ 535.228578] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Expecting reply to msg 18933f9205354c4bb82e1061b36adca7 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 535.401025] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 18933f9205354c4bb82e1061b36adca7 [ 535.411412] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 535.429743] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.429969] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.430136] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 535.466767] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.467342] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.467708] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 535.468680] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.469112] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.469390] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 535.479044] nova-conductor[54394]: DEBUG nova.quota [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Getting quotas for project 520be53772e94d5d9ffaee9642dbe5b4. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 535.481306] nova-conductor[54394]: DEBUG nova.quota [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Getting quotas for user 80543e28baa04ff4a783403666907e03 and project 520be53772e94d5d9ffaee9642dbe5b4. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 535.487956] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] [instance: 783447df-3b87-4df5-8fe3-df137ce9e111] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 535.488703] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.489042] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.489332] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 535.494105] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] [instance: 783447df-3b87-4df5-8fe3-df137ce9e111] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 535.494944] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.496087] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.496403] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 535.514701] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.515106] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.515400] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b600199d-6ba9-4d06-be1c-ae5adf5c83ce tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 535.732712] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 535.769638] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.769857] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.770023] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 535.819398] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.819781] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.820128] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 535.820602] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.821112] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.821182] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 535.838698] nova-conductor[54393]: DEBUG nova.quota [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Getting quotas for project c463c38a01b8412ea32fd3706feeef2d. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 535.841465] nova-conductor[54393]: DEBUG nova.quota [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Getting quotas for user 7768a45e4cf34c1c905094b457fbef99 and project c463c38a01b8412ea32fd3706feeef2d. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 535.846867] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] [instance: eeca9c1a-9d85-4b1c-85e4-c0c2a0a47d4c] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 535.847361] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.847565] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.847730] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 535.855910] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] [instance: eeca9c1a-9d85-4b1c-85e4-c0c2a0a47d4c] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 535.856648] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.856863] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.857026] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 535.895146] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 535.895366] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 535.895544] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 541.136516] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Expecting reply to msg 7697ab77f7cc46afa18475ecfc4cf79f in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 541.311557] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7697ab77f7cc46afa18475ecfc4cf79f [ 541.651190] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Took 0.52 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 541.665792] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 541.666109] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 541.666345] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 541.716377] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 541.716900] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 541.717263] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 541.718062] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 541.718458] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 541.718939] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 541.728132] nova-conductor[54393]: DEBUG nova.quota [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Getting quotas for project 7cc7d365fc69402698470e2821b9a2c2. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 541.732223] nova-conductor[54393]: DEBUG nova.quota [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Getting quotas for user 5ca96abfc71e439d98f27e4b445f3e7a and project 7cc7d365fc69402698470e2821b9a2c2. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 541.745457] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] [instance: e941a033-1429-41e9-9492-d4c692bedf0e] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 541.746030] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 541.746249] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 541.746421] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 541.752497] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] [instance: e941a033-1429-41e9-9492-d4c692bedf0e] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 541.753089] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 541.753355] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 541.753550] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 541.773694] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 541.774045] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 541.774227] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 542.822292] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Expecting reply to msg 200fb39414bd4bb58b00d11ca6c1971a in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 543.003262] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 200fb39414bd4bb58b00d11ca6c1971a [ 543.337656] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Took 0.52 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 543.367101] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 543.367578] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 543.367956] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 543.413947] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 543.414208] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 543.414386] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 543.414861] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 543.415062] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 543.415226] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 543.423495] nova-conductor[54393]: DEBUG nova.quota [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Getting quotas for project 29fcb5b5ee644009a6b7483dba1836d1. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 543.427683] nova-conductor[54393]: DEBUG nova.quota [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Getting quotas for user 676ce5d7595a4b09a1a728128ac12eec and project 29fcb5b5ee644009a6b7483dba1836d1. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 543.434377] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] [instance: b9cbbf4b-efb1-4625-8921-5f4df62f3a43] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 543.434753] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 543.434965] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 543.435140] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 543.446843] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] [instance: b9cbbf4b-efb1-4625-8921-5f4df62f3a43] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 543.447582] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 543.447809] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 543.447965] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 543.469566] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 543.469872] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 543.469973] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 545.381878] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Expecting reply to msg befd51b803e447a885c063fcf92f8b63 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 545.475702] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell0:SELECT=7 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 545.498310] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=19,nova_cell1:SAVEPOINT=6,nova_cell1:INSERT=49,nova_cell1:RELEASE=6 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 545.508171] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=41,nova_api:DELETE=3,nova_api:UPDATE=3 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 545.511077] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=31,nova_api:UPDATE=3,nova_api:DELETE=3 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 545.535320] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg befd51b803e447a885c063fcf92f8b63 [ 545.885453] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 545.908967] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 545.908967] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 545.909102] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 545.949802] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 545.950034] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 545.950199] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 545.950697] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 545.950849] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 545.951184] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 545.960161] nova-conductor[54393]: DEBUG nova.quota [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Getting quotas for project 7f61e591e95c461eba98280093561548. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 545.963123] nova-conductor[54393]: DEBUG nova.quota [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Getting quotas for user 6316bf0499374129a4fa61f170d990c7 and project 7f61e591e95c461eba98280093561548. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 545.968625] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] [instance: 1beb5698-0a25-43af-b89b-23bb888c2b01] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 545.969127] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 545.969326] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 545.969487] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 545.975102] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] [instance: 1beb5698-0a25-43af-b89b-23bb888c2b01] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 545.975783] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 545.976088] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 545.976230] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 546.004088] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 546.004326] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 546.004866] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 546.603336] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Expecting reply to msg 75e9027056fe43b5ba37ef869a6b7be4 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 546.755841] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 75e9027056fe43b5ba37ef869a6b7be4 [ 546.840497] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Expecting reply to msg 5a44a7b4de8d4cd281d9b864dfb6fd35 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 546.993639] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5a44a7b4de8d4cd281d9b864dfb6fd35 [ 547.108964] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 547.130067] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 547.130067] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 547.130067] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 547.186656] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 547.187039] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 547.187811] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 547.187811] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 547.188054] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 547.188964] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 547.202285] nova-conductor[54394]: DEBUG nova.quota [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Getting quotas for project 5a2cb460b72140fe97d5f4fa06385286. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 547.206135] nova-conductor[54394]: DEBUG nova.quota [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Getting quotas for user c8db64d4de5c435c92daeb77e7c1e725 and project 5a2cb460b72140fe97d5f4fa06385286. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 547.218628] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] [instance: 891ed5eb-99fd-47a2-981a-ff7b7cf34be0] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 547.218628] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 547.218628] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 547.218628] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 547.227757] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] [instance: 891ed5eb-99fd-47a2-981a-ff7b7cf34be0] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 547.227757] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 547.227757] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 547.228028] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 547.268112] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 547.268112] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 547.268112] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 547.345903] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 547.370958] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 547.371405] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 547.371740] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 547.452948] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 547.453573] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 547.453573] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 547.453705] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 547.453946] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 547.454065] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 547.469876] nova-conductor[54393]: DEBUG nova.quota [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Getting quotas for project 73d3b459eb7e4f468a43f17271244d98. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 547.474099] nova-conductor[54393]: DEBUG nova.quota [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Getting quotas for user 5d2c0162a05247e6b0697fd9b9a68a6e and project 73d3b459eb7e4f468a43f17271244d98. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 547.482332] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] [instance: 6bbec8d8-9fa5-4787-a570-882065cfe269] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 547.482856] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 547.483065] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 547.483233] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 547.490446] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] [instance: 6bbec8d8-9fa5-4787-a570-882065cfe269] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 547.491146] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 547.491419] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 547.491607] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 547.512803] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 547.513100] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 547.513299] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 547.921198] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] [instance: d20f0dc1-16c6-4d1f-8a50-d3a823343986] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port e8b8757b-7e5d-438b-b109-d3a2d032e1d0, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance d20f0dc1-16c6-4d1f-8a50-d3a823343986 was re-scheduled: Binding failed for port e8b8757b-7e5d-438b-b109-d3a2d032e1d0, please check neutron logs for more information.\n'] [ 547.921881] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 547.922172] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d20f0dc1-16c6-4d1f-8a50-d3a823343986.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d20f0dc1-16c6-4d1f-8a50-d3a823343986. [ 547.922555] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] [instance: d20f0dc1-16c6-4d1f-8a50-d3a823343986] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d20f0dc1-16c6-4d1f-8a50-d3a823343986. [ 547.992995] nova-conductor[54394]: DEBUG nova.network.neutron [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] [instance: d20f0dc1-16c6-4d1f-8a50-d3a823343986] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 548.436424] nova-conductor[54394]: DEBUG nova.network.neutron [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] [instance: d20f0dc1-16c6-4d1f-8a50-d3a823343986] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 548.446338] nova-conductor[54394]: DEBUG nova.network.neutron [None req-70f874ca-2b3b-45f3-8c2a-a24cfcaa099d tempest-FloatingIPsAssociationTestJSON-881154781 tempest-FloatingIPsAssociationTestJSON-881154781-project-member] [instance: d20f0dc1-16c6-4d1f-8a50-d3a823343986] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 550.763889] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Expecting reply to msg 99417cd440004102b9f2d37111ad4aa6 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 550.952205] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 99417cd440004102b9f2d37111ad4aa6 [ 551.278335] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Took 0.52 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 551.297438] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 551.298704] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 551.299216] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 551.347976] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 551.348228] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 551.348396] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 551.349290] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 551.349552] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 551.349724] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 551.366583] nova-conductor[54394]: DEBUG nova.quota [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Getting quotas for project d233cec50aba4ffd88c3cb8395f25d99. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 551.369924] nova-conductor[54394]: DEBUG nova.quota [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Getting quotas for user 040cd65dde9a416dad0bd0a8e4390847 and project d233cec50aba4ffd88c3cb8395f25d99. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 551.387548] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] [instance: 6f7d9bf2-f452-46cf-bd9d-3cb2bafdc82a] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 551.388056] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 551.388274] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 551.388436] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 551.393505] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] [instance: 6f7d9bf2-f452-46cf-bd9d-3cb2bafdc82a] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 551.394174] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 551.394371] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 551.394532] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 551.425101] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 551.425291] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 551.425458] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 553.711200] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Expecting reply to msg 141291b154854aeb983c5acb74c2bb76 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 553.890762] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 141291b154854aeb983c5acb74c2bb76 [ 554.068463] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] [instance: 1c872581-1c03-4724-9dcd-31cb4c270e5d] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 6ab6a0a4-7496-4986-a9f0-87516bee746a, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 1c872581-1c03-4724-9dcd-31cb4c270e5d was re-scheduled: Binding failed for port 6ab6a0a4-7496-4986-a9f0-87516bee746a, please check neutron logs for more information.\n'] [ 554.068684] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 554.069929] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1c872581-1c03-4724-9dcd-31cb4c270e5d.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1c872581-1c03-4724-9dcd-31cb4c270e5d. [ 554.069929] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] [instance: 1c872581-1c03-4724-9dcd-31cb4c270e5d] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1c872581-1c03-4724-9dcd-31cb4c270e5d. [ 554.104294] nova-conductor[54394]: DEBUG nova.network.neutron [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] [instance: 1c872581-1c03-4724-9dcd-31cb4c270e5d] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 554.135807] nova-conductor[54394]: DEBUG nova.network.neutron [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] [instance: 1c872581-1c03-4724-9dcd-31cb4c270e5d] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 554.139595] nova-conductor[54394]: DEBUG nova.network.neutron [None req-b5075f36-8e2c-4ed2-98a2-61e802d10a23 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] [instance: 1c872581-1c03-4724-9dcd-31cb4c270e5d] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 554.168239] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Expecting reply to msg 32b786773ec040b898697f9e93e73ad5 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 554.215134] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 554.233655] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 554.233910] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 554.234093] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 554.289941] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 554.290316] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 554.290528] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 554.290998] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 554.291192] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 554.291323] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 554.308652] nova-conductor[54394]: DEBUG nova.quota [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Getting quotas for project 3ce69c6f908f4f0daac26badbf83c20e. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 554.318278] nova-conductor[54394]: DEBUG nova.quota [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Getting quotas for user 2dbb8985a19844e5bfd019cd0c6a3aa5 and project 3ce69c6f908f4f0daac26badbf83c20e. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 554.336460] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] [instance: 84f10f5a-2757-41fb-81c7-d50e05e8583e] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 554.337829] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 554.338360] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 554.339467] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 554.351662] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] [instance: 84f10f5a-2757-41fb-81c7-d50e05e8583e] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 554.352726] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 554.356469] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 554.358228] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.004s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 554.389099] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 32b786773ec040b898697f9e93e73ad5 [ 554.398104] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 554.398389] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 554.398692] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 554.643185] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Expecting reply to msg 1739ea0a48674cca9cad92dd3a61998b in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 554.671104] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 554.687468] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 554.687468] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 554.687468] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 554.732157] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 554.732401] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 554.732622] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 554.732925] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 554.733111] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 554.733269] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 554.743651] nova-conductor[54394]: DEBUG nova.quota [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Getting quotas for project 296bd969b43b48da85debf428391be90. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 554.749399] nova-conductor[54394]: DEBUG nova.quota [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Getting quotas for user a2b21ec4d62b451c85db372f6b9ff331 and project 296bd969b43b48da85debf428391be90. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 554.760198] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 0c3acf45-7ac5-4eb5-8d5a-6473b6f64252] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 554.760729] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 554.760954] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 554.761128] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 554.765464] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 0c3acf45-7ac5-4eb5-8d5a-6473b6f64252] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 554.766150] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 554.766357] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 554.766524] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 554.784252] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 554.784784] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 554.784784] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 554.794940] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1739ea0a48674cca9cad92dd3a61998b [ 555.149635] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 555.163215] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 555.163547] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 555.163645] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 555.207861] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 555.208168] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 555.208291] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 555.211317] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 555.212037] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 555.212037] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 555.227756] nova-conductor[54394]: DEBUG nova.quota [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Getting quotas for project 71c9e61357c342d194e5839c31c977b5. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 555.231083] nova-conductor[54394]: DEBUG nova.quota [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Getting quotas for user 66e93530fb084c98b0c1747e424d811d and project 71c9e61357c342d194e5839c31c977b5. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 555.236860] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] [instance: db7a244d-b315-4914-bc19-e93e6b762a5d] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 555.237382] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 555.237624] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 555.237804] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 555.255884] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] [instance: db7a244d-b315-4914-bc19-e93e6b762a5d] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 555.256723] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 555.256953] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 555.257116] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 555.283412] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 555.283412] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 555.283412] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 555.580355] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Expecting reply to msg 211fc27991f44ccb8ae7de84955c390b in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 555.772046] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 211fc27991f44ccb8ae7de84955c390b [ 555.877528] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] [instance: aea5f1ef-c521-4419-9d23-f2f3cc53d13a] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port a9dcf9aa-20e2-431f-836d-5e7e474bc6ea, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance aea5f1ef-c521-4419-9d23-f2f3cc53d13a was re-scheduled: Binding failed for port a9dcf9aa-20e2-431f-836d-5e7e474bc6ea, please check neutron logs for more information.\n'] [ 555.878186] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 555.878430] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance aea5f1ef-c521-4419-9d23-f2f3cc53d13a.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance aea5f1ef-c521-4419-9d23-f2f3cc53d13a. [ 555.878762] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] [instance: aea5f1ef-c521-4419-9d23-f2f3cc53d13a] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance aea5f1ef-c521-4419-9d23-f2f3cc53d13a. [ 555.944409] nova-conductor[54394]: DEBUG nova.network.neutron [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] [instance: aea5f1ef-c521-4419-9d23-f2f3cc53d13a] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 556.009676] nova-conductor[54394]: DEBUG nova.network.neutron [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] [instance: aea5f1ef-c521-4419-9d23-f2f3cc53d13a] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 556.022952] nova-conductor[54394]: DEBUG nova.network.neutron [None req-1c597a4e-fd82-4491-b886-65cf89d56282 tempest-ServerDiagnosticsNegativeTest-705117026 tempest-ServerDiagnosticsNegativeTest-705117026-project-member] [instance: aea5f1ef-c521-4419-9d23-f2f3cc53d13a] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 556.085822] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 556.100270] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 556.100517] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 556.100690] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 556.160246] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 556.160550] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 556.160633] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 556.160998] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 556.161172] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 556.161326] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 556.176534] nova-conductor[54394]: DEBUG nova.quota [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Getting quotas for project 3e37435143464ffb8be2f03a9226e14e. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 556.180605] nova-conductor[54394]: DEBUG nova.quota [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Getting quotas for user 6cdec09005b2422e81a03e39e44465d8 and project 3e37435143464ffb8be2f03a9226e14e. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 556.194832] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] [instance: 7b5d282d-d3df-4543-bddd-64901581cfe3] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 556.194832] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 556.194832] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 556.194998] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 556.205566] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] [instance: 7b5d282d-d3df-4543-bddd-64901581cfe3] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 556.206590] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 556.206909] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 556.207192] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 556.230822] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 556.231053] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 556.231221] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 556.608741] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Expecting reply to msg 1f1338d809954f34921ca6982fb5869c in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 556.763431] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1f1338d809954f34921ca6982fb5869c [ 557.112310] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 557.127457] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 557.127741] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 557.127953] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 557.174792] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 557.176913] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.002s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 557.177528] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 557.178174] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 557.178174] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 557.178174] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 557.194098] nova-conductor[54394]: DEBUG nova.quota [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Getting quotas for project 515228b6c9bd4f2185968d863722eac2. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 557.198237] nova-conductor[54394]: DEBUG nova.quota [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Getting quotas for user 730dd9054db2451da1dc1348efac296c and project 515228b6c9bd4f2185968d863722eac2. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 557.217045] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] [instance: 17a076df-2746-4683-bf92-c105e7665cfd] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 557.217045] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 557.217277] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 557.217491] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 557.222424] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] [instance: 17a076df-2746-4683-bf92-c105e7665cfd] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 557.223355] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 557.223689] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 557.224120] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 557.244071] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 557.245145] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 557.245307] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 557.461128] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell0:SELECT=8 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 557.494467] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=23,nova_cell1:SAVEPOINT=7,nova_cell1:INSERT=57,nova_cell1:RELEASE=7 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 557.504057] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=40,nova_api:UPDATE=3,nova_api:DELETE=5 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 557.510050] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=43,nova_api:UPDATE=4,nova_api:DELETE=2 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 559.738812] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Expecting reply to msg 11d463546e284ab1834f79d0aebc47d1 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 559.931865] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 11d463546e284ab1834f79d0aebc47d1 [ 560.065315] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] [instance: ab397815-5da5-4974-8448-b1ffa8d48345] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 034d84cc-0e8a-43e9-b2fc-56473fb6d272, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance ab397815-5da5-4974-8448-b1ffa8d48345 was re-scheduled: Binding failed for port 034d84cc-0e8a-43e9-b2fc-56473fb6d272, please check neutron logs for more information.\n'] [ 560.065901] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 560.066255] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ab397815-5da5-4974-8448-b1ffa8d48345.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ab397815-5da5-4974-8448-b1ffa8d48345. [ 560.066597] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] [instance: ab397815-5da5-4974-8448-b1ffa8d48345] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ab397815-5da5-4974-8448-b1ffa8d48345. [ 560.152591] nova-conductor[54393]: DEBUG nova.network.neutron [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] [instance: ab397815-5da5-4974-8448-b1ffa8d48345] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 560.242103] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 560.259826] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 560.260051] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 560.260275] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 560.333010] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 560.333141] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 560.333311] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 560.333693] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 560.333873] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 560.334027] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 560.351247] nova-conductor[54393]: DEBUG nova.quota [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Getting quotas for project c5d5963725f5417d89259148d2bebbd0. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 560.353634] nova-conductor[54393]: DEBUG nova.quota [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Getting quotas for user 65b5114857ed452ea772a2556009da20 and project c5d5963725f5417d89259148d2bebbd0. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 560.358763] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] [instance: 9d8f3bb7-da42-4a15-a934-8f43b1f90ee1] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 560.359230] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 560.359435] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 560.359607] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 560.365729] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] [instance: 9d8f3bb7-da42-4a15-a934-8f43b1f90ee1] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 560.366399] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 560.366635] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 560.366822] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 560.382870] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 560.383727] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 560.383727] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 560.580173] nova-conductor[54393]: DEBUG nova.network.neutron [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] [instance: ab397815-5da5-4974-8448-b1ffa8d48345] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 560.584195] nova-conductor[54393]: DEBUG nova.network.neutron [None req-e95d13a6-10ef-42e8-af90-c3e1a3568820 tempest-ServerDiagnosticsTest-1800861954 tempest-ServerDiagnosticsTest-1800861954-project-member] [instance: ab397815-5da5-4974-8448-b1ffa8d48345] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 560.721363] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Expecting reply to msg 0a6512b87f8141afafc5002e4183f01a in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 560.862958] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0a6512b87f8141afafc5002e4183f01a [ 561.224681] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 561.246138] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 561.247414] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 561.248657] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 561.296403] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 561.296711] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 561.296887] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 561.297274] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 561.297649] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 561.297811] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 561.300201] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-032ff88f-210e-4598-bf6b-18041d182f8b tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Targeting cell a4379ae1-47d8-4924-9d8e-3d761d765452(cell1) for conductor method rebuild_instance {{(pid=54393) wrapper /opt/stack/nova/nova/conductor/manager.py:95}} [ 561.300424] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-032ff88f-210e-4598-bf6b-18041d182f8b tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 561.300481] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-032ff88f-210e-4598-bf6b-18041d182f8b tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 561.300647] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-032ff88f-210e-4598-bf6b-18041d182f8b tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 561.313921] nova-conductor[54394]: DEBUG nova.quota [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Getting quotas for project df2fd5a65d244a338eb8f63598dac6a0. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 561.319103] nova-conductor[54394]: DEBUG nova.quota [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Getting quotas for user ca4f0f50cbfb4499a8ad482c06431bae and project df2fd5a65d244a338eb8f63598dac6a0. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 561.328163] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-032ff88f-210e-4598-bf6b-18041d182f8b tempest-ServersAdmin275Test-1334644435 tempest-ServersAdmin275Test-1334644435-project-member] [instance: 783447df-3b87-4df5-8fe3-df137ce9e111] No migration record for the rebuild/evacuate request. {{(pid=54393) rebuild_instance /opt/stack/nova/nova/conductor/manager.py:1228}} [ 561.330439] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] [instance: 73139126-8ee3-4c94-b7b2-0e09bab7d1c3] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 561.330845] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 561.331049] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 561.331214] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 561.338902] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] [instance: 73139126-8ee3-4c94-b7b2-0e09bab7d1c3] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 561.339629] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 561.339842] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 561.340032] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 561.378695] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 561.379438] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 561.379710] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 561.892905] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Expecting reply to msg 5e10dffebfb04e2184e0b1eca674e604 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 561.991235] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] [instance: 48bf3159-b4ce-4bbf-b4da-02b824326450] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 4f2100f6-9777-4ff4-a6d7-cb89e9dbdc8a, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 48bf3159-b4ce-4bbf-b4da-02b824326450 was re-scheduled: Binding failed for port 4f2100f6-9777-4ff4-a6d7-cb89e9dbdc8a, please check neutron logs for more information.\n'] [ 561.992570] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 561.993355] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 48bf3159-b4ce-4bbf-b4da-02b824326450.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 48bf3159-b4ce-4bbf-b4da-02b824326450. [ 561.993355] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] [instance: 48bf3159-b4ce-4bbf-b4da-02b824326450] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 48bf3159-b4ce-4bbf-b4da-02b824326450. [ 562.025831] nova-conductor[54394]: DEBUG nova.network.neutron [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] [instance: 48bf3159-b4ce-4bbf-b4da-02b824326450] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 562.109963] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5e10dffebfb04e2184e0b1eca674e604 [ 562.129880] nova-conductor[54394]: DEBUG nova.network.neutron [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] [instance: 48bf3159-b4ce-4bbf-b4da-02b824326450] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 562.133235] nova-conductor[54394]: DEBUG nova.network.neutron [None req-7938ac7c-de3e-4d0f-863c-673a40458c7c tempest-ServerExternalEventsTest-1355747164 tempest-ServerExternalEventsTest-1355747164-project-member] [instance: 48bf3159-b4ce-4bbf-b4da-02b824326450] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 562.401981] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 562.433104] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 562.433333] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 562.433586] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 562.511212] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 562.511542] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 562.511636] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 562.512601] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 562.512601] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 562.512601] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 562.530912] nova-conductor[54394]: DEBUG nova.quota [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Getting quotas for project c0b731d6596043389da6b604c446e52e. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 562.533921] nova-conductor[54394]: DEBUG nova.quota [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Getting quotas for user 7a85c574326b42098620e422a92e5625 and project c0b731d6596043389da6b604c446e52e. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 562.545157] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] [instance: febc3ddb-f569-4ef8-8bb9-73d34f32181a] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 562.545751] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 562.546093] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 562.546319] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 562.552112] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] [instance: febc3ddb-f569-4ef8-8bb9-73d34f32181a] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 562.552846] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 562.553223] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 562.553420] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 562.599350] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 562.599539] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 562.599748] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 570.338121] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell0:SELECT=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 571.294025] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=7,nova_api:UPDATE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 571.331948] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=7,nova_api:DELETE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 571.349279] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=7,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=9,nova_cell1:RELEASE=1,nova_cell1:UPDATE=3 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 572.036523] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Expecting reply to msg e8d865f19fbc417eab85854667e68dc9 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 572.107660] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-5d95689a-1484-46ef-b2ae-86d013a14bdd tempest-ServersAdmin275Test-2072468210 tempest-ServersAdmin275Test-2072468210-project-admin] Targeting cell a4379ae1-47d8-4924-9d8e-3d761d765452(cell1) for conductor method rebuild_instance {{(pid=54393) wrapper /opt/stack/nova/nova/conductor/manager.py:95}} [ 572.107852] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5d95689a-1484-46ef-b2ae-86d013a14bdd tempest-ServersAdmin275Test-2072468210 tempest-ServersAdmin275Test-2072468210-project-admin] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 572.108174] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5d95689a-1484-46ef-b2ae-86d013a14bdd tempest-ServersAdmin275Test-2072468210 tempest-ServersAdmin275Test-2072468210-project-admin] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 572.108266] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5d95689a-1484-46ef-b2ae-86d013a14bdd tempest-ServersAdmin275Test-2072468210 tempest-ServersAdmin275Test-2072468210-project-admin] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 572.132669] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-5d95689a-1484-46ef-b2ae-86d013a14bdd tempest-ServersAdmin275Test-2072468210 tempest-ServersAdmin275Test-2072468210-project-admin] [instance: 783447df-3b87-4df5-8fe3-df137ce9e111] No migration record for the rebuild/evacuate request. {{(pid=54393) rebuild_instance /opt/stack/nova/nova/conductor/manager.py:1228}} [ 572.234138] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e8d865f19fbc417eab85854667e68dc9 [ 572.519664] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell0:SELECT=9 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 572.539909] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 572.555512] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=27,nova_cell1:SAVEPOINT=8,nova_cell1:INSERT=71,nova_cell1:RELEASE=8 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 572.556035] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 572.556396] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 572.556714] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 572.581089] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=57,nova_api:UPDATE=7,nova_api:DELETE=3 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 572.588926] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=51,nova_api:DELETE=6,nova_api:UPDATE=2 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 572.597244] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 572.597491] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 572.597693] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 572.598147] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 572.598361] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 572.598565] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 572.609750] nova-conductor[54393]: DEBUG nova.quota [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Getting quotas for project 4754289a6b174c60af6883bf0bcdc5d2. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 572.613305] nova-conductor[54393]: DEBUG nova.quota [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Getting quotas for user 3ddd435d62a540bbb97e3c8868e6682e and project 4754289a6b174c60af6883bf0bcdc5d2. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 572.619710] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] [instance: c18bc8ab-d411-4340-b150-fae902994f89] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 572.620391] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 572.620501] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 572.620669] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 572.625448] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] [instance: c18bc8ab-d411-4340-b150-fae902994f89] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 572.626185] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 572.626372] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 572.626612] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 572.647378] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 572.647680] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 572.647936] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 574.126985] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=307,nova_cell1:UPDATE=123,nova_cell1:INSERT=25,nova_cell1:SAVEPOINT=29,nova_cell1:RELEASE=29 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 574.482604] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Expecting reply to msg 0e446c66168f41128f78f3bc01a0f834 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 574.534451] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Expecting reply to msg 55844434ae6b4b5d82af4971b868484b in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 574.650203] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0e446c66168f41128f78f3bc01a0f834 [ 574.662545] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] [instance: b2fc6a2b-f913-49c4-8339-9ad8e806dbf0] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port c0c2b6a9-526e-4c41-859c-dab7c04ef4cc, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance b2fc6a2b-f913-49c4-8339-9ad8e806dbf0 was re-scheduled: Binding failed for port c0c2b6a9-526e-4c41-859c-dab7c04ef4cc, please check neutron logs for more information.\n'] [ 574.663245] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 574.663397] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b2fc6a2b-f913-49c4-8339-9ad8e806dbf0.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b2fc6a2b-f913-49c4-8339-9ad8e806dbf0. [ 574.663729] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] [instance: b2fc6a2b-f913-49c4-8339-9ad8e806dbf0] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b2fc6a2b-f913-49c4-8339-9ad8e806dbf0. [ 574.694687] nova-conductor[54394]: DEBUG nova.network.neutron [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] [instance: b2fc6a2b-f913-49c4-8339-9ad8e806dbf0] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 574.741547] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 55844434ae6b4b5d82af4971b868484b [ 574.847003] nova-conductor[54394]: DEBUG nova.network.neutron [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] [instance: b2fc6a2b-f913-49c4-8339-9ad8e806dbf0] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 574.852972] nova-conductor[54394]: DEBUG nova.network.neutron [None req-74109b8d-b591-4126-993c-931d91ae3e14 tempest-ServersNegativeTestMultiTenantJSON-1298641529 tempest-ServersNegativeTestMultiTenantJSON-1298641529-project-member] [instance: b2fc6a2b-f913-49c4-8339-9ad8e806dbf0] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 574.985141] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 575.001483] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 575.001673] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 575.001841] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 575.034104] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 575.034255] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 575.034477] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 575.034854] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 575.035038] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 575.035196] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 575.038171] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 575.046489] nova-conductor[54394]: DEBUG nova.quota [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Getting quotas for project fb849e736e4b4cc2b207d1aae991a29d. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 575.050382] nova-conductor[54394]: DEBUG nova.quota [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Getting quotas for user d122b0d647d64462945ed09817871b99 and project fb849e736e4b4cc2b207d1aae991a29d. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 575.052690] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 575.052969] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 575.053122] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 575.055768] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] [instance: 53baac46-0b8b-438a-905e-cfbb300a7c8d] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 575.056608] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 575.056937] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 575.057226] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 575.062128] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] [instance: 53baac46-0b8b-438a-905e-cfbb300a7c8d] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 575.063083] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 575.063443] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 575.063737] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 575.097055] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 575.097801] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 575.100069] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 575.109859] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 575.110097] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 575.110265] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 575.110633] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 575.110845] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 575.111358] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 575.119737] nova-conductor[54393]: DEBUG nova.quota [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Getting quotas for project 8a36896d8c8543f3b7a91fa671ff3e09. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 575.122028] nova-conductor[54393]: DEBUG nova.quota [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Getting quotas for user 83935c82622b43f4aaec6279dba03c26 and project 8a36896d8c8543f3b7a91fa671ff3e09. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 575.127425] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] [instance: f153e560-4089-4cf3-b87d-dde3a300d101] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 575.127926] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 575.128150] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 575.128309] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 575.131237] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] [instance: f153e560-4089-4cf3-b87d-dde3a300d101] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 575.131853] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 575.132097] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 575.132274] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 575.149624] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 575.149920] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 575.150098] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 575.795357] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Expecting reply to msg 2d749ce60b254325998b0cb75fd0b72f in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 575.959237] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d749ce60b254325998b0cb75fd0b72f [ 576.297887] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Took 0.50 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 576.315698] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 576.315881] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 576.316059] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 576.341933] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Expecting reply to msg 034a9519224345c29387e95a57c9429b in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 576.409079] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 576.409318] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 576.409486] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 576.409870] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 576.410051] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 576.410209] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 576.438603] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] [instance: 7e688ee8-3f7a-42bc-9e2f-bdd850c2c950] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port f9b085be-f596-462d-9060-5654ab4db823, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 7e688ee8-3f7a-42bc-9e2f-bdd850c2c950 was re-scheduled: Binding failed for port f9b085be-f596-462d-9060-5654ab4db823, please check neutron logs for more information.\n'] [ 576.439726] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 576.440020] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7e688ee8-3f7a-42bc-9e2f-bdd850c2c950.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7e688ee8-3f7a-42bc-9e2f-bdd850c2c950. [ 576.440341] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] [instance: 7e688ee8-3f7a-42bc-9e2f-bdd850c2c950] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7e688ee8-3f7a-42bc-9e2f-bdd850c2c950. [ 576.469354] nova-conductor[54394]: DEBUG nova.quota [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Getting quotas for project fb849e736e4b4cc2b207d1aae991a29d. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 576.488598] nova-conductor[54394]: DEBUG nova.quota [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Getting quotas for user d122b0d647d64462945ed09817871b99 and project fb849e736e4b4cc2b207d1aae991a29d. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 576.509226] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] [instance: 24372e6f-8f4b-48a5-b0ac-c1ded8e1fc13] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 576.511076] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 576.512204] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 576.512726] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 576.516610] nova-conductor[54393]: DEBUG nova.network.neutron [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] [instance: 7e688ee8-3f7a-42bc-9e2f-bdd850c2c950] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 576.518782] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] [instance: 24372e6f-8f4b-48a5-b0ac-c1ded8e1fc13] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 576.519687] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 576.520295] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 576.520624] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 576.559415] nova-conductor[54393]: DEBUG nova.network.neutron [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] [instance: 7e688ee8-3f7a-42bc-9e2f-bdd850c2c950] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 576.563232] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 576.563470] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 576.563639] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 576.582409] nova-conductor[54393]: DEBUG nova.network.neutron [None req-674b8455-8504-4049-a9da-7379d479b7dc tempest-ServersTestManualDisk-1731250470 tempest-ServersTestManualDisk-1731250470-project-member] [instance: 7e688ee8-3f7a-42bc-9e2f-bdd850c2c950] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 576.596749] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 034a9519224345c29387e95a57c9429b [ 576.747631] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Expecting reply to msg 5fef3127e5744b66bbb63368c30ac0b8 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 576.844167] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Took 0.50 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 576.856375] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 576.856753] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 576.856981] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 576.911159] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 576.912978] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 576.913175] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 576.913568] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 576.913755] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 576.913985] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 576.937394] nova-conductor[54393]: DEBUG nova.quota [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Getting quotas for project 8a36896d8c8543f3b7a91fa671ff3e09. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 576.940470] nova-conductor[54393]: DEBUG nova.quota [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Getting quotas for user 83935c82622b43f4aaec6279dba03c26 and project 8a36896d8c8543f3b7a91fa671ff3e09. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 576.943449] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5fef3127e5744b66bbb63368c30ac0b8 [ 576.947675] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] [instance: 4b25ccdb-14eb-4e8f-8370-ffd2415b9a2d] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 576.950591] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 576.950591] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 576.950591] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 576.955448] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] [instance: 4b25ccdb-14eb-4e8f-8370-ffd2415b9a2d] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 576.956177] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 576.956402] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 576.956622] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 576.972807] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 576.973078] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 576.973350] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 577.251106] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 577.265640] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 577.269200] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 577.269200] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 577.307578] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 577.307942] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 577.308154] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 577.308656] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 577.308863] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 577.309100] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 577.322287] nova-conductor[54394]: DEBUG nova.quota [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Getting quotas for project 5a9b3125fb214302ade1c00ae3aab4f6. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 577.329017] nova-conductor[54394]: DEBUG nova.quota [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Getting quotas for user 69b987da76bf46599670dd10aa307b0b and project 5a9b3125fb214302ade1c00ae3aab4f6. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 577.338638] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] [instance: 8f6103b2-e7b5-4239-ae75-ced29f1d232a] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 577.339423] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 577.339802] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 577.340107] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 577.344816] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] [instance: 8f6103b2-e7b5-4239-ae75-ced29f1d232a] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 577.345770] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 577.346031] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 577.346282] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 577.369267] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 577.369602] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 577.369824] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 577.747691] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Expecting reply to msg 3aaff8aa7706499ea17ca1748e7f982a in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 577.923909] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3aaff8aa7706499ea17ca1748e7f982a [ 578.255532] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 578.289894] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 578.289894] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 578.289894] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 578.339599] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 578.339849] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 578.340063] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 578.340497] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 578.340625] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 578.340741] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 578.366342] nova-conductor[54393]: DEBUG nova.quota [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Getting quotas for project ecca28e310194e36907da0aacbf9bce3. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 578.371107] nova-conductor[54393]: DEBUG nova.quota [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Getting quotas for user a74d160d8ab04669a535e3f633a644f4 and project ecca28e310194e36907da0aacbf9bce3. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 578.380231] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] [instance: eabf4498-97a5-4994-a0f9-431415602a92] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 578.380720] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 578.380937] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 578.381958] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 578.394497] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] [instance: eabf4498-97a5-4994-a0f9-431415602a92] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 578.395315] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 578.395455] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 578.395626] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 578.423235] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 578.423235] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 578.423235] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 578.924348] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=352,nova_cell1:UPDATE=160,nova_cell1:INSERT=18,nova_cell1:SAVEPOINT=22,nova_cell1:RELEASE=22 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 579.494853] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Expecting reply to msg 24f2b5b28708463b8430accf26ea8e00 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 579.673403] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 24f2b5b28708463b8430accf26ea8e00 [ 579.998421] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 580.013045] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 580.013317] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 580.013492] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 580.062981] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 580.063147] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 580.063316] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 580.063691] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 580.063855] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 580.064042] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 580.072980] nova-conductor[54394]: DEBUG nova.quota [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Getting quotas for project 7b6177ee33c74f3a89ab5251a94038e7. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 580.076348] nova-conductor[54394]: DEBUG nova.quota [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Getting quotas for user d1429ed3916c42af92467fd7255cb2ec and project 7b6177ee33c74f3a89ab5251a94038e7. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 580.085330] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: fb6ef31f-ac48-4f03-8047-73cead0e3af7] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 580.085863] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 580.086060] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 580.086239] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 580.091973] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: fb6ef31f-ac48-4f03-8047-73cead0e3af7] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 580.092705] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 580.092956] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 580.093161] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 580.111143] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 580.111578] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 580.111857] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 580.978331] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] [instance: eeca9c1a-9d85-4b1c-85e4-c0c2a0a47d4c] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 0d58231c-a8d4-41fd-86cf-3ea6ff711598, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance eeca9c1a-9d85-4b1c-85e4-c0c2a0a47d4c was re-scheduled: Binding failed for port 0d58231c-a8d4-41fd-86cf-3ea6ff711598, please check neutron logs for more information.\n'] [ 580.979124] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 580.979413] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance eeca9c1a-9d85-4b1c-85e4-c0c2a0a47d4c.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance eeca9c1a-9d85-4b1c-85e4-c0c2a0a47d4c. [ 580.979998] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] [instance: eeca9c1a-9d85-4b1c-85e4-c0c2a0a47d4c] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance eeca9c1a-9d85-4b1c-85e4-c0c2a0a47d4c. [ 581.006244] nova-conductor[54394]: DEBUG nova.network.neutron [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] [instance: eeca9c1a-9d85-4b1c-85e4-c0c2a0a47d4c] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 581.053250] nova-conductor[54394]: DEBUG nova.network.neutron [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] [instance: eeca9c1a-9d85-4b1c-85e4-c0c2a0a47d4c] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 581.057473] nova-conductor[54394]: DEBUG nova.network.neutron [None req-e9caefdc-8401-4123-ac86-efbb3b1cc6b1 tempest-AttachInterfacesV270Test-744438739 tempest-AttachInterfacesV270Test-744438739-project-member] [instance: eeca9c1a-9d85-4b1c-85e4-c0c2a0a47d4c] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 585.245587] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Expecting reply to msg 0bd89ad6b3474e23b11fa6169ca7a890 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 585.351018] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] [instance: e941a033-1429-41e9-9492-d4c692bedf0e] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port b0a721b6-c77b-471c-b18c-d663781313a0, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance e941a033-1429-41e9-9492-d4c692bedf0e was re-scheduled: Binding failed for port b0a721b6-c77b-471c-b18c-d663781313a0, please check neutron logs for more information.\n'] [ 585.351513] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 585.351736] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e941a033-1429-41e9-9492-d4c692bedf0e.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e941a033-1429-41e9-9492-d4c692bedf0e. [ 585.352058] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] [instance: e941a033-1429-41e9-9492-d4c692bedf0e] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e941a033-1429-41e9-9492-d4c692bedf0e. [ 585.378300] nova-conductor[54393]: DEBUG nova.network.neutron [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] [instance: e941a033-1429-41e9-9492-d4c692bedf0e] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 585.403262] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0bd89ad6b3474e23b11fa6169ca7a890 [ 585.477043] nova-conductor[54393]: DEBUG nova.network.neutron [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] [instance: e941a033-1429-41e9-9492-d4c692bedf0e] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 585.480888] nova-conductor[54393]: DEBUG nova.network.neutron [None req-9281399b-6c57-45d2-847f-24522c0c0ea3 tempest-ServersWithSpecificFlavorTestJSON-601929680 tempest-ServersWithSpecificFlavorTestJSON-601929680-project-member] [instance: e941a033-1429-41e9-9492-d4c692bedf0e] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 585.748916] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 585.762414] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 585.762646] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 585.762821] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 585.810524] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 585.811003] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 585.812064] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 585.812620] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 585.812966] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 585.813276] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 585.825748] nova-conductor[54393]: DEBUG nova.quota [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Getting quotas for project aa9c82f3916646a3b0b3ecaaa915a267. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 585.829794] nova-conductor[54393]: DEBUG nova.quota [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Getting quotas for user 625b7228e273485a9b5ac953c1fb1af4 and project aa9c82f3916646a3b0b3ecaaa915a267. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 585.835979] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] [instance: 9e8a52be-389a-4cd1-8838-4adedc750463] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 585.836638] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 585.837007] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 585.837326] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 585.845765] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] [instance: 9e8a52be-389a-4cd1-8838-4adedc750463] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 585.846652] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 585.847340] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 585.847691] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 585.874371] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 585.874776] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 585.875019] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 587.272297] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Expecting reply to msg a152bfbeb7074936badfa370aa7b5d89 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 587.422801] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a152bfbeb7074936badfa370aa7b5d89 [ 587.775915] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 587.795776] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 587.795938] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 587.796124] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 587.879376] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 587.879791] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 587.880127] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 587.880692] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 587.880998] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 587.881277] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 587.898792] nova-conductor[54394]: DEBUG nova.quota [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Getting quotas for project 0165f78eb05842f9b32679dfa5d9a456. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 587.903904] nova-conductor[54394]: DEBUG nova.quota [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Getting quotas for user 981d0bdb139e4e00b80a43d9ebf40b9c and project 0165f78eb05842f9b32679dfa5d9a456. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 587.909569] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] [instance: d810fd33-4fdf-4761-ad65-cc7bfc9d38ec] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 587.910059] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 587.910285] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 587.910567] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 587.921859] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] [instance: d810fd33-4fdf-4761-ad65-cc7bfc9d38ec] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 587.922405] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 587.922724] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 587.922949] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 587.954998] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 587.955224] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 587.955388] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 589.254092] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Expecting reply to msg 4d1588bf1f4947e99bea205d7d8bd805 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 589.454079] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4d1588bf1f4947e99bea205d7d8bd805 [ 589.763154] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 589.776975] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 589.777348] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 589.778576] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 589.812636] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 589.813230] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 589.813524] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 589.814165] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 589.814483] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 589.814876] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 589.825867] nova-conductor[54393]: DEBUG nova.quota [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Getting quotas for project 94445eb0173f4436a97e542b3e4ab25c. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 589.829137] nova-conductor[54393]: DEBUG nova.quota [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Getting quotas for user 0749e213ae08467b8c05b62ff1a5bde4 and project 94445eb0173f4436a97e542b3e4ab25c. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 589.834893] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] [instance: 61b2069d-160d-474b-a7ba-40d66d340509] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 589.835706] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 589.836049] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 589.836336] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 589.846179] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] [instance: 61b2069d-160d-474b-a7ba-40d66d340509] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 589.848172] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 589.848491] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 589.848842] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 589.894864] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 589.895541] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 589.895905] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 594.665390] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] [instance: b9cbbf4b-efb1-4625-8921-5f4df62f3a43] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 84bcc27c-31f9-4684-84c6-28c3e2c06f43, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance b9cbbf4b-efb1-4625-8921-5f4df62f3a43 was re-scheduled: Binding failed for port 84bcc27c-31f9-4684-84c6-28c3e2c06f43, please check neutron logs for more information.\n'] [ 594.665979] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 594.666135] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b9cbbf4b-efb1-4625-8921-5f4df62f3a43.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b9cbbf4b-efb1-4625-8921-5f4df62f3a43. [ 594.666441] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] [instance: b9cbbf4b-efb1-4625-8921-5f4df62f3a43] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b9cbbf4b-efb1-4625-8921-5f4df62f3a43. [ 594.704345] nova-conductor[54393]: DEBUG nova.network.neutron [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] [instance: b9cbbf4b-efb1-4625-8921-5f4df62f3a43] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 594.759292] nova-conductor[54393]: DEBUG nova.network.neutron [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] [instance: b9cbbf4b-efb1-4625-8921-5f4df62f3a43] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 594.762492] nova-conductor[54393]: DEBUG nova.network.neutron [None req-db27b70b-77fb-41a7-8bcd-df5b3a6d4181 tempest-TenantUsagesTestJSON-1297943457 tempest-TenantUsagesTestJSON-1297943457-project-member] [instance: b9cbbf4b-efb1-4625-8921-5f4df62f3a43] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 596.011448] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Expecting reply to msg 12982c0cc1fe43e9913abf5dfedd8eb9 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 596.154452] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 12982c0cc1fe43e9913abf5dfedd8eb9 [ 596.514675] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 596.538041] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 596.538041] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 596.538280] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 596.643863] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 596.644297] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 596.644708] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 596.645511] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 596.645810] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 596.646066] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 596.662881] nova-conductor[54393]: DEBUG nova.quota [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Getting quotas for project 85d9d77976814640b7f1a62b1425f212. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 596.667232] nova-conductor[54393]: DEBUG nova.quota [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Getting quotas for user 7688143b948f48afbb6c69681b0e8847 and project 85d9d77976814640b7f1a62b1425f212. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 596.673219] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] [instance: cec498c8-a45a-422d-9bde-45b317a0c05a] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 596.673684] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 596.673886] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 596.674047] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 596.678031] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] [instance: cec498c8-a45a-422d-9bde-45b317a0c05a] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 596.678702] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 596.678904] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 596.679067] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 596.696438] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 596.696696] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 596.696933] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 597.886588] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell0:SELECT=5 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 597.926094] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=15,nova_cell1:SAVEPOINT=4,nova_cell1:INSERT=39,nova_cell1:RELEASE=4 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 597.950359] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=27,nova_api:UPDATE=4,nova_api:DELETE=3 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 597.952482] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=33,nova_api:DELETE=2,nova_api:UPDATE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 598.880487] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Expecting reply to msg 0cdf55de795542b986489d9c55009386 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 599.045687] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0cdf55de795542b986489d9c55009386 [ 599.382905] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 599.403745] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 599.404055] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 599.404265] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 599.467642] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 599.467867] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 599.468059] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 599.468436] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 599.468643] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 599.468782] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 599.485884] nova-conductor[54394]: DEBUG nova.quota [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Getting quotas for project 1a46b925d9ca4d6a829e5f53635c400d. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 599.489542] nova-conductor[54394]: DEBUG nova.quota [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Getting quotas for user 6eb61f0c108d417cbe0d38a1202fe993 and project 1a46b925d9ca4d6a829e5f53635c400d. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 599.498191] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] [instance: e3aedb85-c405-430e-882d-7a5d4be22616] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 599.499116] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 599.499116] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 599.499116] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 599.506811] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] [instance: e3aedb85-c405-430e-882d-7a5d4be22616] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 599.507617] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 599.507896] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 599.508039] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 599.523652] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 599.523880] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 599.524068] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 602.840753] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] [instance: 1beb5698-0a25-43af-b89b-23bb888c2b01] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 9638788d-bfb9-4c2c-a7e5-573877bb7da3, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 1beb5698-0a25-43af-b89b-23bb888c2b01 was re-scheduled: Binding failed for port 9638788d-bfb9-4c2c-a7e5-573877bb7da3, please check neutron logs for more information.\n'] [ 602.841516] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 602.843917] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1beb5698-0a25-43af-b89b-23bb888c2b01.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1beb5698-0a25-43af-b89b-23bb888c2b01. [ 602.845945] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] [instance: 1beb5698-0a25-43af-b89b-23bb888c2b01] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1beb5698-0a25-43af-b89b-23bb888c2b01. [ 602.900782] nova-conductor[54393]: DEBUG nova.network.neutron [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] [instance: 1beb5698-0a25-43af-b89b-23bb888c2b01] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 602.949380] nova-conductor[54393]: DEBUG nova.network.neutron [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] [instance: 1beb5698-0a25-43af-b89b-23bb888c2b01] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 602.955013] nova-conductor[54393]: DEBUG nova.network.neutron [None req-782d38ca-3316-4fa9-bc7c-d2abaaf5ceb9 tempest-VolumesAssistedSnapshotsTest-1253851209 tempest-VolumesAssistedSnapshotsTest-1253851209-project-member] [instance: 1beb5698-0a25-43af-b89b-23bb888c2b01] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 603.245225] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Expecting reply to msg 4d7908ceaaca46789184ef4bde0003c1 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 603.425537] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4d7908ceaaca46789184ef4bde0003c1 [ 603.717031] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Expecting reply to msg 78ffb26644394d9faaf7e411d0233b52 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 603.749104] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 603.761925] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 603.762154] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 603.762411] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 603.825203] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 603.825440] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 603.825602] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 603.825967] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 603.826185] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 603.826301] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 603.835478] nova-conductor[54394]: DEBUG nova.quota [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Getting quotas for project 35015aaa974c4265b0db7d1b7e617ff1. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 603.837888] nova-conductor[54394]: DEBUG nova.quota [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Getting quotas for user 12b5b6d4e37846a791b610cc8afb8979 and project 35015aaa974c4265b0db7d1b7e617ff1. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 603.843549] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] [instance: 155db01a-6594-4e26-9bdd-f46d783fc27b] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 603.844071] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 603.844275] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 603.844437] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 603.860224] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] [instance: 155db01a-6594-4e26-9bdd-f46d783fc27b] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 603.860944] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 603.861147] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 603.861310] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 603.881224] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 78ffb26644394d9faaf7e411d0233b52 [ 603.895974] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 603.897195] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 603.897195] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 604.219915] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 604.235724] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 604.236044] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 604.236332] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 604.404298] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 604.404545] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 604.404723] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 604.405076] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 604.405259] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 604.405455] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 604.434838] nova-conductor[54394]: DEBUG nova.quota [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Getting quotas for project 68e908be63a74ce6b4f2751efa7b8dd5. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 604.446022] nova-conductor[54394]: DEBUG nova.quota [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Getting quotas for user e0630c88affc44fc8a99d3318e324260 and project 68e908be63a74ce6b4f2751efa7b8dd5. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 604.457584] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] [instance: 14bcf4eb-5f36-43b8-b838-1152fd7f4007] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 604.458164] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 604.458368] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 604.458535] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 604.462820] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] [instance: 14bcf4eb-5f36-43b8-b838-1152fd7f4007] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 604.463538] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 604.463703] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 604.463867] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 604.488791] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 604.488791] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 604.488791] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-9c6c1de5-3132-4a4c-b192-713b81c6fe1d tempest-ServerDiagnosticsV248Test-2094172881 tempest-ServerDiagnosticsV248Test-2094172881-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 604.794798] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Expecting reply to msg 60bd6ea4bee6424ea6509d64e5ec9ec0 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 604.937593] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 60bd6ea4bee6424ea6509d64e5ec9ec0 [ 605.298146] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 605.312499] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 605.312796] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 605.313023] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 605.375060] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 605.375473] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 605.375742] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 605.376900] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 605.377369] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 605.377654] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 605.396446] nova-conductor[54393]: DEBUG nova.quota [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Getting quotas for project 30b45d69f8c9449ebdbef0e386be6a21. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 605.404714] nova-conductor[54393]: DEBUG nova.quota [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Getting quotas for user bb88a29c241347348248ca37d1f76cd1 and project 30b45d69f8c9449ebdbef0e386be6a21. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 605.425024] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] [instance: 17b1a6d1-3ae0-4b0d-ba01-ef98a355a04b] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 605.425908] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 605.426720] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 605.427045] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 605.435670] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] [instance: 17b1a6d1-3ae0-4b0d-ba01-ef98a355a04b] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 605.437700] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 605.437700] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 605.437700] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 605.471108] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 605.471327] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 605.471494] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 610.925329] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] [instance: 891ed5eb-99fd-47a2-981a-ff7b7cf34be0] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 2ff602f8-ed2a-401e-ad43-fd3bfb177af6, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 891ed5eb-99fd-47a2-981a-ff7b7cf34be0 was re-scheduled: Binding failed for port 2ff602f8-ed2a-401e-ad43-fd3bfb177af6, please check neutron logs for more information.\n'] [ 610.926055] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 610.926300] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 891ed5eb-99fd-47a2-981a-ff7b7cf34be0.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 891ed5eb-99fd-47a2-981a-ff7b7cf34be0. [ 610.926647] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] [instance: 891ed5eb-99fd-47a2-981a-ff7b7cf34be0] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 891ed5eb-99fd-47a2-981a-ff7b7cf34be0. [ 610.951117] nova-conductor[54393]: DEBUG nova.network.neutron [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] [instance: 891ed5eb-99fd-47a2-981a-ff7b7cf34be0] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 610.970932] nova-conductor[54393]: DEBUG nova.network.neutron [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] [instance: 891ed5eb-99fd-47a2-981a-ff7b7cf34be0] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 610.974713] nova-conductor[54393]: DEBUG nova.network.neutron [None req-5be54e24-82b1-46a5-9035-ffd9815b6ed8 tempest-ServersAdminNegativeTestJSON-700888499 tempest-ServersAdminNegativeTestJSON-700888499-project-member] [instance: 891ed5eb-99fd-47a2-981a-ff7b7cf34be0] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 613.053276] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] [instance: 6bbec8d8-9fa5-4787-a570-882065cfe269] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 42dfcb53-9a4e-45f5-a0a8-978967e15b01, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 6bbec8d8-9fa5-4787-a570-882065cfe269 was re-scheduled: Binding failed for port 42dfcb53-9a4e-45f5-a0a8-978967e15b01, please check neutron logs for more information.\n'] [ 613.054342] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 613.054583] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 6bbec8d8-9fa5-4787-a570-882065cfe269.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 6bbec8d8-9fa5-4787-a570-882065cfe269. [ 613.054791] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] [instance: 6bbec8d8-9fa5-4787-a570-882065cfe269] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 6bbec8d8-9fa5-4787-a570-882065cfe269. [ 613.082582] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] [instance: 6bbec8d8-9fa5-4787-a570-882065cfe269] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 613.099943] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] [instance: 6bbec8d8-9fa5-4787-a570-882065cfe269] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 613.103224] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d6fa4d39-b030-4ba9-955d-002a99e47cb1 tempest-ServerAddressesTestJSON-714163622 tempest-ServerAddressesTestJSON-714163622-project-member] [instance: 6bbec8d8-9fa5-4787-a570-882065cfe269] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 613.331419] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Expecting reply to msg b986e65c923d438592b798071b6fe164 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 613.484902] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b986e65c923d438592b798071b6fe164 [ 613.839367] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 613.853661] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.853908] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.854095] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.926060] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.926315] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.926407] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.926846] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.927030] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.927223] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.938829] nova-conductor[54393]: DEBUG nova.quota [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Getting quotas for project 7480e31795e743c493b3f5bc6e1db072. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 613.942304] nova-conductor[54393]: DEBUG nova.quota [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Getting quotas for user 7e678faa61fd4961a6a2955a835fdbc4 and project 7480e31795e743c493b3f5bc6e1db072. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 613.954728] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] [instance: ef5c2b13-e9d6-48a2-b56f-8b84eb62a6eb] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 613.955258] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.955463] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.955625] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.958676] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] [instance: ef5c2b13-e9d6-48a2-b56f-8b84eb62a6eb] block_device_mapping [BlockDeviceMapping(attachment_id=04a33560-2105-4f32-9bb2-6100636bede5,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='volume',tag=None,updated_at=,uuid=,volume_id='f5e6c079-4e1f-48fa-8ded-27f6b3321ee9',volume_size=1,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 613.959395] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.959636] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.959804] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.975965] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.976235] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.976405] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 614.409611] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell0:SELECT=3 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 614.466404] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=9,nova_cell1:SAVEPOINT=3,nova_cell1:INSERT=25,nova_cell1:RELEASE=3 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 614.475845] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=16,nova_api:UPDATE=2,nova_api:DELETE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 614.479365] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=20,nova_api:UPDATE=1,nova_api:DELETE=2 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 615.083582] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] [instance: 6f7d9bf2-f452-46cf-bd9d-3cb2bafdc82a] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 58e56efd-7dd7-4385-9432-8fca66f6ee2c, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 6f7d9bf2-f452-46cf-bd9d-3cb2bafdc82a was re-scheduled: Binding failed for port 58e56efd-7dd7-4385-9432-8fca66f6ee2c, please check neutron logs for more information.\n'] [ 615.084330] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 615.084704] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 6f7d9bf2-f452-46cf-bd9d-3cb2bafdc82a.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 6f7d9bf2-f452-46cf-bd9d-3cb2bafdc82a. [ 615.085109] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] [instance: 6f7d9bf2-f452-46cf-bd9d-3cb2bafdc82a] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 6f7d9bf2-f452-46cf-bd9d-3cb2bafdc82a. [ 615.124637] nova-conductor[54393]: DEBUG nova.network.neutron [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] [instance: 6f7d9bf2-f452-46cf-bd9d-3cb2bafdc82a] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 615.145332] nova-conductor[54393]: DEBUG nova.network.neutron [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] [instance: 6f7d9bf2-f452-46cf-bd9d-3cb2bafdc82a] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 615.150008] nova-conductor[54393]: DEBUG nova.network.neutron [None req-c928f41d-ccdf-47a3-8889-bfd0e03d6bfd tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] [instance: 6f7d9bf2-f452-46cf-bd9d-3cb2bafdc82a] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 616.810959] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Expecting reply to msg acac26d58878427f834b102ec31a7f2d in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 616.971703] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg acac26d58878427f834b102ec31a7f2d [ 617.315315] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 617.330867] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 617.331138] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 617.331449] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 617.383436] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 617.383676] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 617.383859] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 617.384248] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 617.384423] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 617.384587] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 617.410971] nova-conductor[54393]: DEBUG nova.quota [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Getting quotas for project d233cec50aba4ffd88c3cb8395f25d99. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 617.414877] nova-conductor[54393]: DEBUG nova.quota [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Getting quotas for user 040cd65dde9a416dad0bd0a8e4390847 and project d233cec50aba4ffd88c3cb8395f25d99. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 617.437748] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] [instance: 76913c20-992d-4233-b596-3ffc16533f1c] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 617.438350] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 617.438560] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 617.438731] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 617.443773] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] [instance: 76913c20-992d-4233-b596-3ffc16533f1c] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 617.444480] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 617.444686] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 617.444851] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 617.461732] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 617.461949] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 617.462114] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 618.069173] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Expecting reply to msg f3ec709ccb32432c9dae2ee461bb95da in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 618.218595] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3ec709ccb32432c9dae2ee461bb95da [ 618.572220] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 618.588847] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 618.590180] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 618.590180] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 618.622752] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 618.622974] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 618.623194] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 618.623596] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 618.623776] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 618.623928] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 618.633967] nova-conductor[54393]: DEBUG nova.quota [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Getting quotas for project 9bc3b936a0c740ee92e4b015dffe8fb8. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 618.637140] nova-conductor[54393]: DEBUG nova.quota [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Getting quotas for user a5cf6d8c76b8455b841a9c462ebaae21 and project 9bc3b936a0c740ee92e4b015dffe8fb8. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 618.642799] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] [instance: c61c3592-c849-4a92-ac58-1455b4350ae3] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 618.643300] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 618.643512] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 618.643683] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 618.649475] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] [instance: c61c3592-c849-4a92-ac58-1455b4350ae3] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 618.650275] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 618.650487] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 618.650659] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 618.667536] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 618.667798] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 618.667975] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cb5eb24e-14a0-4774-ba25-9db0e99c8260 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 621.630086] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] [instance: 84f10f5a-2757-41fb-81c7-d50e05e8583e] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port bf52d290-1165-4d81-9b54-6ed240f44727, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 84f10f5a-2757-41fb-81c7-d50e05e8583e was re-scheduled: Binding failed for port bf52d290-1165-4d81-9b54-6ed240f44727, please check neutron logs for more information.\n'] [ 621.630729] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 621.630964] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 84f10f5a-2757-41fb-81c7-d50e05e8583e.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 84f10f5a-2757-41fb-81c7-d50e05e8583e. [ 621.631278] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] [instance: 84f10f5a-2757-41fb-81c7-d50e05e8583e] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 84f10f5a-2757-41fb-81c7-d50e05e8583e. [ 621.656311] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] [instance: 84f10f5a-2757-41fb-81c7-d50e05e8583e] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 621.673270] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] [instance: 84f10f5a-2757-41fb-81c7-d50e05e8583e] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 621.677213] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d328eff6-c975-4c02-bdfb-96dbc6db3276 tempest-InstanceActionsV221TestJSON-1058527641 tempest-InstanceActionsV221TestJSON-1058527641-project-member] [instance: 84f10f5a-2757-41fb-81c7-d50e05e8583e] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 622.075515] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Expecting reply to msg 264aa4e410684517b25fe548b173ec72 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 622.198985] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 264aa4e410684517b25fe548b173ec72 [ 622.578675] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 622.593046] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 622.593313] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 622.593511] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 622.632261] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 622.632560] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 622.632775] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 622.633145] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 622.633361] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 622.633538] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 622.679166] nova-conductor[54393]: DEBUG nova.quota [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Getting quotas for project beefcf89e47443d29fab14a11e91a3cf. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 622.682787] nova-conductor[54393]: DEBUG nova.quota [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Getting quotas for user 87a6014a457d4c58b551a7402599680c and project beefcf89e47443d29fab14a11e91a3cf. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 622.696217] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] [instance: 89363983-217b-4137-b46e-fd53ebcde670] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 622.696750] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 622.696984] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 622.697194] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 622.701371] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] [instance: 89363983-217b-4137-b46e-fd53ebcde670] block_device_mapping [BlockDeviceMapping(attachment_id=32c31e53-d75f-407e-8826-64fcb7551696,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='volume',tag=None,updated_at=,uuid=,volume_id='4f982543-4519-4469-a434-d4f7d5d24e29',volume_size=1,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 622.702205] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 622.705759] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 622.705990] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.004s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 622.724224] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 622.724433] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 622.724600] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 623.699647] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 0c3acf45-7ac5-4eb5-8d5a-6473b6f64252] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port c965b411-1109-4cf4-b7d9-4b9bcbc2e0ee, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 0c3acf45-7ac5-4eb5-8d5a-6473b6f64252 was re-scheduled: Binding failed for port c965b411-1109-4cf4-b7d9-4b9bcbc2e0ee, please check neutron logs for more information.\n'] [ 623.700303] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 623.700537] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 0c3acf45-7ac5-4eb5-8d5a-6473b6f64252.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 0c3acf45-7ac5-4eb5-8d5a-6473b6f64252. [ 623.700850] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 0c3acf45-7ac5-4eb5-8d5a-6473b6f64252] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 0c3acf45-7ac5-4eb5-8d5a-6473b6f64252. [ 623.735147] nova-conductor[54393]: DEBUG nova.network.neutron [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 0c3acf45-7ac5-4eb5-8d5a-6473b6f64252] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 623.756754] nova-conductor[54393]: DEBUG nova.network.neutron [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 0c3acf45-7ac5-4eb5-8d5a-6473b6f64252] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 623.760797] nova-conductor[54393]: DEBUG nova.network.neutron [None req-c6469be4-19df-4ac1-ad91-04c00a228210 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 0c3acf45-7ac5-4eb5-8d5a-6473b6f64252] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 625.679427] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Expecting reply to msg 4f52af0998f2422d9fa8a25930c3948b in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 625.856328] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4f52af0998f2422d9fa8a25930c3948b [ 626.184370] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 626.201772] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 626.202070] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 626.202207] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 626.243626] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 626.243845] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 626.244063] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 626.244430] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 626.244610] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 626.244953] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 626.263917] nova-conductor[54393]: DEBUG nova.quota [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Getting quotas for project 296bd969b43b48da85debf428391be90. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 626.270162] nova-conductor[54393]: DEBUG nova.quota [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Getting quotas for user a2b21ec4d62b451c85db372f6b9ff331 and project 296bd969b43b48da85debf428391be90. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 626.277782] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 4c8b8d30-b619-4f48-90df-6343923b14ba] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 626.278294] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 626.278499] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 626.278704] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 626.295171] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 4c8b8d30-b619-4f48-90df-6343923b14ba] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 626.295930] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 626.296165] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 626.296373] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 626.326435] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 626.326801] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 626.326975] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 626.719167] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Expecting reply to msg 7def6fb3d3aa4714b747b092286dbc3a in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 626.863407] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7def6fb3d3aa4714b747b092286dbc3a [ 627.167424] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Expecting reply to msg a024735b14ba42f5bcb732b0998b95cb in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 627.223467] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 627.252558] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 627.252823] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 627.253062] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 627.328940] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 627.329185] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 627.329355] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 627.329734] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 627.329916] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 627.330078] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 627.333998] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a024735b14ba42f5bcb732b0998b95cb [ 627.340601] nova-conductor[54393]: DEBUG nova.quota [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Getting quotas for project 2846f5b19081440298e3f4a2cf97a8ba. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 627.343641] nova-conductor[54393]: DEBUG nova.quota [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Getting quotas for user 24aa47fb771d4d7aa8313ff63d8313f0 and project 2846f5b19081440298e3f4a2cf97a8ba. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 627.352956] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: 45450786-0b10-40b9-8564-1bd4d417f0d6] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 627.353747] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 627.353971] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 627.354141] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 627.359052] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: 45450786-0b10-40b9-8564-1bd4d417f0d6] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 627.359726] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 627.359995] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 627.360185] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 627.388259] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 627.388497] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 627.388666] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 627.668304] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 627.687545] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 627.688045] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 627.688162] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 627.724550] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Expecting reply to msg 302cbab246c94580a18c8f3ba9997d1a in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 627.729859] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 627.730152] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 627.731165] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 627.731165] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 627.731165] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 627.731165] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 627.752032] nova-conductor[54394]: DEBUG nova.quota [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Getting quotas for project 2846f5b19081440298e3f4a2cf97a8ba. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 627.752211] nova-conductor[54394]: DEBUG nova.quota [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Getting quotas for user 24aa47fb771d4d7aa8313ff63d8313f0 and project 2846f5b19081440298e3f4a2cf97a8ba. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 627.758013] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: d9770d22-0b9e-4d18-bcec-68cc2b491480] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 627.758502] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 627.758736] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 627.758931] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 627.766268] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: d9770d22-0b9e-4d18-bcec-68cc2b491480] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 627.768842] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 627.770731] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 627.770946] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.002s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 627.797436] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 627.797634] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 627.797804] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 627.867911] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 302cbab246c94580a18c8f3ba9997d1a [ 628.228149] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 628.233512] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] [instance: db7a244d-b315-4914-bc19-e93e6b762a5d] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port b90f52b7-8d6f-4683-ba82-f0c2bba8a094, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance db7a244d-b315-4914-bc19-e93e6b762a5d was re-scheduled: Binding failed for port b90f52b7-8d6f-4683-ba82-f0c2bba8a094, please check neutron logs for more information.\n'] [ 628.234288] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 628.234464] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance db7a244d-b315-4914-bc19-e93e6b762a5d.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance db7a244d-b315-4914-bc19-e93e6b762a5d. [ 628.234817] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] [instance: db7a244d-b315-4914-bc19-e93e6b762a5d] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance db7a244d-b315-4914-bc19-e93e6b762a5d. [ 628.259126] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 628.259388] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 628.259535] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 628.273474] nova-conductor[54393]: DEBUG nova.network.neutron [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] [instance: db7a244d-b315-4914-bc19-e93e6b762a5d] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 628.299505] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 628.299661] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 628.300019] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 628.300406] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 628.301237] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 628.301237] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 628.308761] nova-conductor[54393]: DEBUG nova.network.neutron [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] [instance: db7a244d-b315-4914-bc19-e93e6b762a5d] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 628.316710] nova-conductor[54393]: DEBUG nova.network.neutron [None req-b921c4ea-20ca-45de-b7a3-99a942109c07 tempest-ImagesNegativeTestJSON-1040281020 tempest-ImagesNegativeTestJSON-1040281020-project-member] [instance: db7a244d-b315-4914-bc19-e93e6b762a5d] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 628.316977] nova-conductor[54394]: DEBUG nova.quota [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Getting quotas for project 2846f5b19081440298e3f4a2cf97a8ba. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 628.320923] nova-conductor[54394]: DEBUG nova.quota [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Getting quotas for user 24aa47fb771d4d7aa8313ff63d8313f0 and project 2846f5b19081440298e3f4a2cf97a8ba. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 628.330959] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: 5db8dac7-6f97-4684-a147-bafee0e180e8] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 628.330959] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 628.330959] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 628.330959] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 628.340087] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: 5db8dac7-6f97-4684-a147-bafee0e180e8] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 628.340967] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 628.341192] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 628.341361] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 628.364495] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 628.365029] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 628.365203] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 630.190247] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] [instance: 7b5d282d-d3df-4543-bddd-64901581cfe3] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 62ca7a1f-5a7c-495f-a17f-177f8eb4b23a, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 7b5d282d-d3df-4543-bddd-64901581cfe3 was re-scheduled: Binding failed for port 62ca7a1f-5a7c-495f-a17f-177f8eb4b23a, please check neutron logs for more information.\n'] [ 630.190862] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 630.191103] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7b5d282d-d3df-4543-bddd-64901581cfe3.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7b5d282d-d3df-4543-bddd-64901581cfe3. [ 630.191413] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] [instance: 7b5d282d-d3df-4543-bddd-64901581cfe3] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7b5d282d-d3df-4543-bddd-64901581cfe3. [ 630.225765] nova-conductor[54394]: DEBUG nova.network.neutron [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] [instance: 7b5d282d-d3df-4543-bddd-64901581cfe3] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 630.244242] nova-conductor[54394]: DEBUG nova.network.neutron [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] [instance: 7b5d282d-d3df-4543-bddd-64901581cfe3] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 630.247464] nova-conductor[54394]: DEBUG nova.network.neutron [None req-4ded7783-0c9a-408b-a441-d7c40b35d3b1 tempest-ImagesOneServerNegativeTestJSON-90680537 tempest-ImagesOneServerNegativeTestJSON-90680537-project-member] [instance: 7b5d282d-d3df-4543-bddd-64901581cfe3] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 632.191593] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] [instance: 17a076df-2746-4683-bf92-c105e7665cfd] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 1281b2e2-65cf-4bb1-9266-cc32add8ce4f, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 17a076df-2746-4683-bf92-c105e7665cfd was re-scheduled: Binding failed for port 1281b2e2-65cf-4bb1-9266-cc32add8ce4f, please check neutron logs for more information.\n'] [ 632.191593] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 632.192184] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 17a076df-2746-4683-bf92-c105e7665cfd.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 17a076df-2746-4683-bf92-c105e7665cfd. [ 632.192184] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] [instance: 17a076df-2746-4683-bf92-c105e7665cfd] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 17a076df-2746-4683-bf92-c105e7665cfd. [ 632.215197] nova-conductor[54394]: DEBUG nova.network.neutron [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] [instance: 17a076df-2746-4683-bf92-c105e7665cfd] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 632.265471] nova-conductor[54394]: DEBUG nova.network.neutron [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] [instance: 17a076df-2746-4683-bf92-c105e7665cfd] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 632.269953] nova-conductor[54394]: DEBUG nova.network.neutron [None req-7570bb7b-25e4-4e86-9e7c-47926efadf34 tempest-FloatingIPsAssociationNegativeTestJSON-661683152 tempest-FloatingIPsAssociationNegativeTestJSON-661683152-project-member] [instance: 17a076df-2746-4683-bf92-c105e7665cfd] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 634.135486] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=378,nova_cell1:UPDATE=190,nova_cell1:INSERT=22,nova_cell1:SAVEPOINT=36,nova_cell1:RELEASE=36,nova_cell1:DELETE=2 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 637.333684] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell0:SELECT=14 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 637.362807] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=46,nova_cell1:UPDATE=3,nova_cell1:INSERT=111,nova_cell1:SAVEPOINT=11,nova_cell1:RELEASE=11 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 637.377479] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=79,nova_api:UPDATE=7,nova_api:DELETE=6 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 637.382802] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=91,nova_api:UPDATE=7,nova_api:DELETE=8 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 638.304869] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell0:SELECT=2 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 638.343533] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=6,nova_cell1:INSERT=14 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 638.352855] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 638.359499] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1,nova_api:DELETE=2 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 638.578860] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Expecting reply to msg 5000b3cfd0af403f9d9728417a8b196e in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 638.743560] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5000b3cfd0af403f9d9728417a8b196e [ 639.090640] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 639.104118] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 639.104522] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 639.104827] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 639.167624] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 639.167987] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 639.168116] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 639.168546] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 639.168877] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 639.169096] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 639.177870] nova-conductor[54394]: DEBUG nova.quota [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Getting quotas for project 84f92b8fdb454f5ea89482c54f629b6d. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 639.180486] nova-conductor[54394]: DEBUG nova.quota [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Getting quotas for user 1655c5595da44fdfaa0d4b5fd309a2b5 and project 84f92b8fdb454f5ea89482c54f629b6d. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 639.186732] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] [instance: 1a1954da-5b61-42ea-8474-4c3ea1d6d31d] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 639.186732] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 639.192565] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 639.192856] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.006s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 639.205829] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] [instance: 1a1954da-5b61-42ea-8474-4c3ea1d6d31d] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 639.205943] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 639.206401] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 639.206550] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 639.243807] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 639.247395] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 639.247749] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.004s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 639.257273] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=383,nova_cell1:SAVEPOINT=39,nova_cell1:RELEASE=39,nova_cell1:UPDATE=178,nova_cell1:INSERT=31 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 641.901445] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] [instance: 9d8f3bb7-da42-4a15-a934-8f43b1f90ee1] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 922de905-a9f7-42e0-b35d-aeb9f47e58e6, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 9d8f3bb7-da42-4a15-a934-8f43b1f90ee1 was re-scheduled: Binding failed for port 922de905-a9f7-42e0-b35d-aeb9f47e58e6, please check neutron logs for more information.\n'] [ 641.902312] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 641.902560] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 9d8f3bb7-da42-4a15-a934-8f43b1f90ee1.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 9d8f3bb7-da42-4a15-a934-8f43b1f90ee1. [ 641.902866] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] [instance: 9d8f3bb7-da42-4a15-a934-8f43b1f90ee1] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 9d8f3bb7-da42-4a15-a934-8f43b1f90ee1. [ 641.928328] nova-conductor[54394]: DEBUG nova.network.neutron [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] [instance: 9d8f3bb7-da42-4a15-a934-8f43b1f90ee1] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 641.943742] nova-conductor[54394]: DEBUG nova.network.neutron [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] [instance: 9d8f3bb7-da42-4a15-a934-8f43b1f90ee1] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 641.949074] nova-conductor[54394]: DEBUG nova.network.neutron [None req-5a99509e-7b83-4d06-b9a6-1b557e6d5fd7 tempest-ServersV294TestFqdnHostnames-545730839 tempest-ServersV294TestFqdnHostnames-545730839-project-member] [instance: 9d8f3bb7-da42-4a15-a934-8f43b1f90ee1] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 645.729534] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 645.730122] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 645.730422] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 645.756047] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Expecting reply to msg a931a2e15a884e6ba23115f57d7b21d4 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 645.924282] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a931a2e15a884e6ba23115f57d7b21d4 [ 646.259237] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 646.283814] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 646.284064] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 646.284235] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 646.347260] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 646.347260] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 646.347396] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 646.348105] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 646.348105] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 646.348214] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 646.362213] nova-conductor[54394]: DEBUG nova.quota [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Getting quotas for project 041b00f08104400a9378f1205b7a0b89. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 646.370230] nova-conductor[54394]: DEBUG nova.quota [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Getting quotas for user 0b33cb2192d14d21942aa7cd44714904 and project 041b00f08104400a9378f1205b7a0b89. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 646.375790] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] [instance: 3940fe7c-561b-4ee7-9e36-9aa24d400a38] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 646.376338] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 646.376508] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 646.376770] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 646.383523] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] [instance: 3940fe7c-561b-4ee7-9e36-9aa24d400a38] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 646.384224] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 646.384671] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 646.384671] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 646.405410] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 646.405640] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 646.405879] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 649.068623] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] [instance: 73139126-8ee3-4c94-b7b2-0e09bab7d1c3] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port da239a6b-db92-40b8-be3c-e75fb0fc0890, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 73139126-8ee3-4c94-b7b2-0e09bab7d1c3 was re-scheduled: Binding failed for port da239a6b-db92-40b8-be3c-e75fb0fc0890, please check neutron logs for more information.\n'] [ 649.069220] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 649.069451] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 73139126-8ee3-4c94-b7b2-0e09bab7d1c3.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 73139126-8ee3-4c94-b7b2-0e09bab7d1c3. [ 649.069769] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] [instance: 73139126-8ee3-4c94-b7b2-0e09bab7d1c3] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 73139126-8ee3-4c94-b7b2-0e09bab7d1c3. [ 649.134028] nova-conductor[54394]: DEBUG nova.network.neutron [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] [instance: 73139126-8ee3-4c94-b7b2-0e09bab7d1c3] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 649.150716] nova-conductor[54394]: DEBUG nova.network.neutron [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] [instance: 73139126-8ee3-4c94-b7b2-0e09bab7d1c3] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 649.154528] nova-conductor[54394]: DEBUG nova.network.neutron [None req-334c0d0a-a16e-4686-974a-c50008beecb2 tempest-DeleteServersAdminTestJSON-1927797331 tempest-DeleteServersAdminTestJSON-1927797331-project-member] [instance: 73139126-8ee3-4c94-b7b2-0e09bab7d1c3] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 650.186790] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Expecting reply to msg f3c2d0e1fc0e40e7bb8f8a9e15c071d5 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 650.448425] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f3c2d0e1fc0e40e7bb8f8a9e15c071d5 [ 650.689651] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 650.706696] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 650.706961] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 650.707137] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 650.747438] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 650.747675] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 650.747848] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 650.748242] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 650.748434] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 650.748594] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 650.762320] nova-conductor[54393]: DEBUG nova.quota [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Getting quotas for project cdaf5fdfb3d5461cb787538ff6b0962a. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 650.765878] nova-conductor[54393]: DEBUG nova.quota [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Getting quotas for user a6ca13e3d6e844d4a70cb2c740a689e8 and project cdaf5fdfb3d5461cb787538ff6b0962a. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 650.770851] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] [instance: d3ea9b40-256d-4e90-8d13-49dc08766d00] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 650.771303] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 650.771506] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 650.771674] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 650.776679] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] [instance: d3ea9b40-256d-4e90-8d13-49dc08766d00] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 650.777393] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 650.777532] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 650.777739] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 650.804346] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 650.804346] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 650.804346] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 656.352996] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell0:SELECT=2 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 656.386964] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=6,nova_cell1:SAVEPOINT=2,nova_cell1:INSERT=16,nova_cell1:RELEASE=2 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 656.398381] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=14,nova_api:UPDATE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 656.403073] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1,nova_api:DELETE=2 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 658.708341] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] [instance: febc3ddb-f569-4ef8-8bb9-73d34f32181a] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 35a6b292-85bb-4d3c-bc40-047b44172da8, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance febc3ddb-f569-4ef8-8bb9-73d34f32181a was re-scheduled: Binding failed for port 35a6b292-85bb-4d3c-bc40-047b44172da8, please check neutron logs for more information.\n'] [ 658.708941] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 658.709169] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance febc3ddb-f569-4ef8-8bb9-73d34f32181a.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance febc3ddb-f569-4ef8-8bb9-73d34f32181a. [ 658.709377] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] [instance: febc3ddb-f569-4ef8-8bb9-73d34f32181a] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance febc3ddb-f569-4ef8-8bb9-73d34f32181a. [ 658.729504] nova-conductor[54394]: DEBUG nova.network.neutron [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] [instance: febc3ddb-f569-4ef8-8bb9-73d34f32181a] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 658.743950] nova-conductor[54394]: DEBUG nova.network.neutron [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] [instance: febc3ddb-f569-4ef8-8bb9-73d34f32181a] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 658.747773] nova-conductor[54394]: DEBUG nova.network.neutron [None req-8bcc1e25-76b6-45ef-8cfe-16bcf4f8b8e0 tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] [instance: febc3ddb-f569-4ef8-8bb9-73d34f32181a] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 660.014937] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Expecting reply to msg bbb6a0ed3b4944f2a3831364bb3ec0d7 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 660.172158] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bbb6a0ed3b4944f2a3831364bb3ec0d7 [ 660.517705] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 660.532780] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 660.532780] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 660.532780] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 660.579638] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 660.579879] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 660.584200] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 660.584200] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 660.584200] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 660.584200] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 660.598588] nova-conductor[54393]: DEBUG nova.quota [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Getting quotas for project c0b731d6596043389da6b604c446e52e. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 660.602371] nova-conductor[54393]: DEBUG nova.quota [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Getting quotas for user 7a85c574326b42098620e422a92e5625 and project c0b731d6596043389da6b604c446e52e. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 660.609374] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] [instance: 74b49da4-a792-42fa-9f01-19c17e898786] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 660.609891] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 660.610097] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 660.610283] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 660.622628] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] [instance: 74b49da4-a792-42fa-9f01-19c17e898786] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 660.623509] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 660.623750] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 660.623888] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 660.627440] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] [instance: c18bc8ab-d411-4340-b150-fae902994f89] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 11bfd5e4-1313-4a3b-95c5-caa9de786f37, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance c18bc8ab-d411-4340-b150-fae902994f89 was re-scheduled: Binding failed for port 11bfd5e4-1313-4a3b-95c5-caa9de786f37, please check neutron logs for more information.\n'] [ 660.631718] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 660.631972] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance c18bc8ab-d411-4340-b150-fae902994f89.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance c18bc8ab-d411-4340-b150-fae902994f89. [ 660.632257] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] [instance: c18bc8ab-d411-4340-b150-fae902994f89] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance c18bc8ab-d411-4340-b150-fae902994f89. [ 660.659230] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 660.659230] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 660.659230] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 660.673397] nova-conductor[54394]: DEBUG nova.network.neutron [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] [instance: c18bc8ab-d411-4340-b150-fae902994f89] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 660.689096] nova-conductor[54394]: DEBUG nova.network.neutron [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] [instance: c18bc8ab-d411-4340-b150-fae902994f89] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 660.692950] nova-conductor[54394]: DEBUG nova.network.neutron [None req-b9376bd0-9576-41e4-aa9d-9c3fbad7cea6 tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] [instance: c18bc8ab-d411-4340-b150-fae902994f89] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 662.484343] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] [instance: 53baac46-0b8b-438a-905e-cfbb300a7c8d] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port a4648984-6a21-4581-9379-fe30374891fc, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 53baac46-0b8b-438a-905e-cfbb300a7c8d was re-scheduled: Binding failed for port a4648984-6a21-4581-9379-fe30374891fc, please check neutron logs for more information.\n'] [ 662.485432] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 662.485669] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 53baac46-0b8b-438a-905e-cfbb300a7c8d.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 53baac46-0b8b-438a-905e-cfbb300a7c8d. [ 662.485876] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] [instance: 53baac46-0b8b-438a-905e-cfbb300a7c8d] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 53baac46-0b8b-438a-905e-cfbb300a7c8d. [ 662.514850] nova-conductor[54394]: DEBUG nova.network.neutron [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] [instance: 53baac46-0b8b-438a-905e-cfbb300a7c8d] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 662.534688] nova-conductor[54394]: DEBUG nova.network.neutron [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] [instance: 53baac46-0b8b-438a-905e-cfbb300a7c8d] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 662.539813] nova-conductor[54394]: DEBUG nova.network.neutron [None req-a7c0bd5a-895e-4c33-b114-ae0192305b15 tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] [instance: 53baac46-0b8b-438a-905e-cfbb300a7c8d] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 663.756486] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Expecting reply to msg a318dc5a7c114788b19861e47fbe22f2 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 663.899336] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a318dc5a7c114788b19861e47fbe22f2 [ 664.255755] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 664.268878] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 664.269109] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 664.269273] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 664.294799] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 664.295042] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 664.295214] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 664.295614] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 664.295817] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 664.296033] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 664.303334] nova-conductor[54394]: DEBUG nova.quota [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Getting quotas for project 4754289a6b174c60af6883bf0bcdc5d2. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 664.305474] nova-conductor[54394]: DEBUG nova.quota [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Getting quotas for user 3ddd435d62a540bbb97e3c8868e6682e and project 4754289a6b174c60af6883bf0bcdc5d2. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 664.310467] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] [instance: 602c12e8-b5b5-4969-93d0-1f9fb8c464ec] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 664.310895] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 664.311090] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 664.311251] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 664.314875] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] [instance: 602c12e8-b5b5-4969-93d0-1f9fb8c464ec] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 664.315486] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 664.315673] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 664.315832] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 664.333317] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 664.333555] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 664.333724] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 666.992874] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] [instance: f153e560-4089-4cf3-b87d-dde3a300d101] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 6fd0e059-2f59-494f-ba92-2a0a0df11838, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance f153e560-4089-4cf3-b87d-dde3a300d101 was re-scheduled: Binding failed for port 6fd0e059-2f59-494f-ba92-2a0a0df11838, please check neutron logs for more information.\n'] [ 666.992874] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 666.992874] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance f153e560-4089-4cf3-b87d-dde3a300d101.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance f153e560-4089-4cf3-b87d-dde3a300d101. [ 666.992874] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] [instance: f153e560-4089-4cf3-b87d-dde3a300d101] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance f153e560-4089-4cf3-b87d-dde3a300d101. [ 666.992874] nova-conductor[54394]: DEBUG nova.network.neutron [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] [instance: f153e560-4089-4cf3-b87d-dde3a300d101] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 667.001164] nova-conductor[54394]: DEBUG nova.network.neutron [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] [instance: f153e560-4089-4cf3-b87d-dde3a300d101] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 667.004599] nova-conductor[54394]: DEBUG nova.network.neutron [None req-2d857643-75a6-41e6-994e-f7b39171ba4c tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] [instance: f153e560-4089-4cf3-b87d-dde3a300d101] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 668.951747] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] [instance: 24372e6f-8f4b-48a5-b0ac-c1ded8e1fc13] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port dd93863d-9975-4d37-b7a7-5d0d5c3cea30, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 24372e6f-8f4b-48a5-b0ac-c1ded8e1fc13 was re-scheduled: Binding failed for port dd93863d-9975-4d37-b7a7-5d0d5c3cea30, please check neutron logs for more information.\n'] [ 668.952389] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 668.952627] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 24372e6f-8f4b-48a5-b0ac-c1ded8e1fc13.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 24372e6f-8f4b-48a5-b0ac-c1ded8e1fc13. [ 668.952959] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] [instance: 24372e6f-8f4b-48a5-b0ac-c1ded8e1fc13] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 24372e6f-8f4b-48a5-b0ac-c1ded8e1fc13. [ 668.974617] nova-conductor[54393]: DEBUG nova.network.neutron [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] [instance: 24372e6f-8f4b-48a5-b0ac-c1ded8e1fc13] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 668.989447] nova-conductor[54393]: DEBUG nova.network.neutron [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] [instance: 24372e6f-8f4b-48a5-b0ac-c1ded8e1fc13] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 668.993010] nova-conductor[54393]: DEBUG nova.network.neutron [None req-3bbe0194-27e0-477f-af60-6ff0affb58fc tempest-ListImageFiltersTestJSON-1952177359 tempest-ListImageFiltersTestJSON-1952177359-project-member] [instance: 24372e6f-8f4b-48a5-b0ac-c1ded8e1fc13] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 670.585548] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell0:SELECT=2 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 670.627146] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=6,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=15,nova_cell1:RELEASE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 670.645928] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=9,nova_api:UPDATE=2,nova_api:DELETE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 670.650757] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=15,nova_api:DELETE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 674.013129] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Expecting reply to msg 77835ded6ab14934a674c97a04b1bb3e in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 674.157308] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 77835ded6ab14934a674c97a04b1bb3e [ 674.299651] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell0:SELECT=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 674.317758] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=3,nova_cell1:INSERT=7 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 674.327138] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=7,nova_api:UPDATE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 674.329946] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=5,nova_api:DELETE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 674.518633] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 674.531081] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 674.531547] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 674.531931] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 674.572441] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 674.572960] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 674.573323] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 674.573913] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 674.574358] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 674.574704] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 674.585208] nova-conductor[54393]: DEBUG nova.quota [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Getting quotas for project ad7980bce21246ccb289123f979e9039. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 674.588946] nova-conductor[54393]: DEBUG nova.quota [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Getting quotas for user a847c2a768364ec8836bea167952cec5 and project ad7980bce21246ccb289123f979e9039. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 674.595493] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: 30053ec6-5706-4908-831a-5af650b391b6] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 674.596337] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 674.596750] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 674.597096] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 674.606953] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: 30053ec6-5706-4908-831a-5af650b391b6] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 674.608506] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 674.608900] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 674.609243] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 674.625276] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 674.625783] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 674.626184] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 684.580431] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell0:SELECT=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 684.612423] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=3,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=8,nova_cell1:RELEASE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 684.620702] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=7 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 684.622915] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=5,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 687.262726] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] [instance: 4b25ccdb-14eb-4e8f-8370-ffd2415b9a2d] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 7bf89d71-17ea-48d1-bc24-73a809922e74, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 4b25ccdb-14eb-4e8f-8370-ffd2415b9a2d was re-scheduled: Binding failed for port 7bf89d71-17ea-48d1-bc24-73a809922e74, please check neutron logs for more information.\n'] [ 687.263302] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 687.263598] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 4b25ccdb-14eb-4e8f-8370-ffd2415b9a2d.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 4b25ccdb-14eb-4e8f-8370-ffd2415b9a2d. [ 687.263954] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] [instance: 4b25ccdb-14eb-4e8f-8370-ffd2415b9a2d] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 4b25ccdb-14eb-4e8f-8370-ffd2415b9a2d. [ 687.291212] nova-conductor[54393]: DEBUG nova.network.neutron [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] [instance: 4b25ccdb-14eb-4e8f-8370-ffd2415b9a2d] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 687.312417] nova-conductor[54393]: DEBUG nova.network.neutron [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] [instance: 4b25ccdb-14eb-4e8f-8370-ffd2415b9a2d] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 687.315397] nova-conductor[54393]: DEBUG nova.network.neutron [None req-13c47eb7-04c5-4329-b2c9-02fa50819eef tempest-ServersAdminTestJSON-2055785351 tempest-ServersAdminTestJSON-2055785351-project-member] [instance: 4b25ccdb-14eb-4e8f-8370-ffd2415b9a2d] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 689.155922] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] [instance: 8f6103b2-e7b5-4239-ae75-ced29f1d232a] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 91bc138f-c601-45a5-9110-f5f26da43027, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 8f6103b2-e7b5-4239-ae75-ced29f1d232a was re-scheduled: Binding failed for port 91bc138f-c601-45a5-9110-f5f26da43027, please check neutron logs for more information.\n'] [ 689.156506] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 689.156763] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8f6103b2-e7b5-4239-ae75-ced29f1d232a.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8f6103b2-e7b5-4239-ae75-ced29f1d232a. [ 689.156985] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] [instance: 8f6103b2-e7b5-4239-ae75-ced29f1d232a] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8f6103b2-e7b5-4239-ae75-ced29f1d232a. [ 689.194966] nova-conductor[54394]: DEBUG nova.network.neutron [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] [instance: 8f6103b2-e7b5-4239-ae75-ced29f1d232a] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 689.240364] nova-conductor[54394]: DEBUG nova.network.neutron [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] [instance: 8f6103b2-e7b5-4239-ae75-ced29f1d232a] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 689.243696] nova-conductor[54394]: DEBUG nova.network.neutron [None req-d9a67e6c-8fbb-431a-a82c-c4211ab246bf tempest-InstanceActionsTestJSON-398043156 tempest-InstanceActionsTestJSON-398043156-project-member] [instance: 8f6103b2-e7b5-4239-ae75-ced29f1d232a] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 691.136430] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] [instance: eabf4498-97a5-4994-a0f9-431415602a92] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 80f3ef03-d161-4ee4-a3eb-844a23eec1f3, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance eabf4498-97a5-4994-a0f9-431415602a92 was re-scheduled: Binding failed for port 80f3ef03-d161-4ee4-a3eb-844a23eec1f3, please check neutron logs for more information.\n'] [ 691.137036] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 691.137256] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance eabf4498-97a5-4994-a0f9-431415602a92.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance eabf4498-97a5-4994-a0f9-431415602a92. [ 691.137459] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] [instance: eabf4498-97a5-4994-a0f9-431415602a92] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance eabf4498-97a5-4994-a0f9-431415602a92. [ 691.170510] nova-conductor[54393]: DEBUG nova.network.neutron [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] [instance: eabf4498-97a5-4994-a0f9-431415602a92] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 691.201682] nova-conductor[54393]: DEBUG nova.network.neutron [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] [instance: eabf4498-97a5-4994-a0f9-431415602a92] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 691.205885] nova-conductor[54393]: DEBUG nova.network.neutron [None req-ebb84b91-0866-488a-ac36-545eb35e2bbc tempest-ServerRescueTestJSON-1499512551 tempest-ServerRescueTestJSON-1499512551-project-member] [instance: eabf4498-97a5-4994-a0f9-431415602a92] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 694.331447] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=322,nova_cell1:UPDATE=151,nova_cell1:SAVEPOINT=31,nova_cell1:RELEASE=31,nova_cell1:INSERT=26 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 698.514182] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Expecting reply to msg eb9054e62a4446cc9e76e5f5fef13da2 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 698.674757] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg eb9054e62a4446cc9e76e5f5fef13da2 [ 699.016098] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Took 0.50 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 699.028818] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 699.029047] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 699.029214] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 699.054130] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 699.054349] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 699.054519] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 699.054875] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 699.055236] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 699.055426] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 699.062520] nova-conductor[54394]: DEBUG nova.quota [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Getting quotas for project 6f72f3af363643548c0b8198679e32d2. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 699.064610] nova-conductor[54394]: DEBUG nova.quota [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Getting quotas for user dced843e8f57438abc6fa19e9be16ba4 and project 6f72f3af363643548c0b8198679e32d2. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 699.069412] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] [instance: ef61895f-75c8-449d-a5e8-b9b4d6a97a45] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 699.070304] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 699.070304] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 699.070304] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 699.077858] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] [instance: ef61895f-75c8-449d-a5e8-b9b4d6a97a45] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 699.078473] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 699.078684] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 699.078856] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 699.094551] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 699.094727] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 699.094882] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 699.462506] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=272,nova_cell1:UPDATE=131,nova_cell1:INSERT=15,nova_cell1:SAVEPOINT=31,nova_cell1:RELEASE=31 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 700.499192] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: fb6ef31f-ac48-4f03-8047-73cead0e3af7] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port bc1bad50-601f-48c7-aee7-2b4ba5b6a14d, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance fb6ef31f-ac48-4f03-8047-73cead0e3af7 was re-scheduled: Binding failed for port bc1bad50-601f-48c7-aee7-2b4ba5b6a14d, please check neutron logs for more information.\n'] [ 700.499815] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 700.500557] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance fb6ef31f-ac48-4f03-8047-73cead0e3af7.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance fb6ef31f-ac48-4f03-8047-73cead0e3af7. [ 700.500821] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: fb6ef31f-ac48-4f03-8047-73cead0e3af7] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance fb6ef31f-ac48-4f03-8047-73cead0e3af7. [ 700.524203] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: fb6ef31f-ac48-4f03-8047-73cead0e3af7] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 700.549597] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: fb6ef31f-ac48-4f03-8047-73cead0e3af7] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 700.552869] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d33066fd-466b-4eed-9ae3-11db4bfd12f1 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: fb6ef31f-ac48-4f03-8047-73cead0e3af7] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 701.837470] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Expecting reply to msg 2d3dca155a974836b120346c0b7ac741 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 701.996877] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2d3dca155a974836b120346c0b7ac741 [ 702.339685] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Took 0.50 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 702.352033] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 702.352033] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 702.352033] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 702.382576] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 702.382802] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 702.382977] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 702.383346] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 702.383523] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 702.383678] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 702.391756] nova-conductor[54394]: DEBUG nova.quota [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Getting quotas for project 7b6177ee33c74f3a89ab5251a94038e7. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 702.394734] nova-conductor[54394]: DEBUG nova.quota [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Getting quotas for user d1429ed3916c42af92467fd7255cb2ec and project 7b6177ee33c74f3a89ab5251a94038e7. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 702.399661] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: e10acb93-1ed9-4c73-a84d-ac0ae213755d] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 702.400135] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 702.400333] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 702.400496] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 702.404715] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: e10acb93-1ed9-4c73-a84d-ac0ae213755d] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 702.405302] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 702.405498] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 702.405663] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 702.419807] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 702.420027] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 702.420203] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 707.140142] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] [instance: 9e8a52be-389a-4cd1-8838-4adedc750463] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port cc8ae856-5903-425d-ae10-91e265a7e85c, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 9e8a52be-389a-4cd1-8838-4adedc750463 was re-scheduled: Binding failed for port cc8ae856-5903-425d-ae10-91e265a7e85c, please check neutron logs for more information.\n'] [ 707.140142] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 707.140142] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 9e8a52be-389a-4cd1-8838-4adedc750463.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 9e8a52be-389a-4cd1-8838-4adedc750463. [ 707.140142] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] [instance: 9e8a52be-389a-4cd1-8838-4adedc750463] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 9e8a52be-389a-4cd1-8838-4adedc750463. [ 707.160309] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] [instance: 9e8a52be-389a-4cd1-8838-4adedc750463] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 707.180670] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] [instance: 9e8a52be-389a-4cd1-8838-4adedc750463] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 707.193500] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d3052111-07a5-4fe2-a719-4c5455aaca9a tempest-ServerActionsTestOtherA-548991697 tempest-ServerActionsTestOtherA-548991697-project-member] [instance: 9e8a52be-389a-4cd1-8838-4adedc750463] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 712.387334] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell0:SELECT=2 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 712.407564] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=6,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=15,nova_cell1:RELEASE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 712.414518] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 712.418361] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=12,nova_api:UPDATE=1,nova_api:DELETE=2 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 716.278698] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] [instance: d810fd33-4fdf-4761-ad65-cc7bfc9d38ec] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 0b54c045-2023-40d7-b3c1-128049dfba9c, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance d810fd33-4fdf-4761-ad65-cc7bfc9d38ec was re-scheduled: Binding failed for port 0b54c045-2023-40d7-b3c1-128049dfba9c, please check neutron logs for more information.\n'] [ 716.279716] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 716.279716] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d810fd33-4fdf-4761-ad65-cc7bfc9d38ec.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d810fd33-4fdf-4761-ad65-cc7bfc9d38ec. [ 716.280504] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] [instance: d810fd33-4fdf-4761-ad65-cc7bfc9d38ec] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d810fd33-4fdf-4761-ad65-cc7bfc9d38ec. [ 716.303358] nova-conductor[54393]: DEBUG nova.network.neutron [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] [instance: d810fd33-4fdf-4761-ad65-cc7bfc9d38ec] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 716.317006] nova-conductor[54393]: DEBUG nova.network.neutron [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] [instance: d810fd33-4fdf-4761-ad65-cc7bfc9d38ec] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 716.320380] nova-conductor[54393]: DEBUG nova.network.neutron [None req-2426cb3c-5975-44f6-8987-5695ce448c78 tempest-AttachInterfacesUnderV243Test-870201315 tempest-AttachInterfacesUnderV243Test-870201315-project-member] [instance: d810fd33-4fdf-4761-ad65-cc7bfc9d38ec] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 718.234772] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] [instance: 61b2069d-160d-474b-a7ba-40d66d340509] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 430f770d-6f25-4349-bfe3-74ac94b154e8, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 61b2069d-160d-474b-a7ba-40d66d340509 was re-scheduled: Binding failed for port 430f770d-6f25-4349-bfe3-74ac94b154e8, please check neutron logs for more information.\n'] [ 718.235464] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 718.235694] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 61b2069d-160d-474b-a7ba-40d66d340509.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 61b2069d-160d-474b-a7ba-40d66d340509. [ 718.235904] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] [instance: 61b2069d-160d-474b-a7ba-40d66d340509] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 61b2069d-160d-474b-a7ba-40d66d340509. [ 718.272477] nova-conductor[54393]: DEBUG nova.network.neutron [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] [instance: 61b2069d-160d-474b-a7ba-40d66d340509] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 718.293210] nova-conductor[54393]: DEBUG nova.network.neutron [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] [instance: 61b2069d-160d-474b-a7ba-40d66d340509] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 718.296369] nova-conductor[54393]: DEBUG nova.network.neutron [None req-0f304a21-0d11-4c19-81e1-abc9f21a2e69 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] [instance: 61b2069d-160d-474b-a7ba-40d66d340509] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 720.147905] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] [instance: cec498c8-a45a-422d-9bde-45b317a0c05a] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port b4e1e1c0-f739-4dc3-a2ab-a87347b155fd, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance cec498c8-a45a-422d-9bde-45b317a0c05a was re-scheduled: Binding failed for port b4e1e1c0-f739-4dc3-a2ab-a87347b155fd, please check neutron logs for more information.\n'] [ 720.147905] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 720.147905] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance cec498c8-a45a-422d-9bde-45b317a0c05a.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance cec498c8-a45a-422d-9bde-45b317a0c05a. [ 720.147905] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] [instance: cec498c8-a45a-422d-9bde-45b317a0c05a] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance cec498c8-a45a-422d-9bde-45b317a0c05a. [ 720.180733] nova-conductor[54393]: DEBUG nova.network.neutron [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] [instance: cec498c8-a45a-422d-9bde-45b317a0c05a] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 720.206419] nova-conductor[54393]: DEBUG nova.network.neutron [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] [instance: cec498c8-a45a-422d-9bde-45b317a0c05a] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 720.212212] nova-conductor[54393]: DEBUG nova.network.neutron [None req-1c762a08-3d56-495f-93c5-305549953cde tempest-ServerAddressesNegativeTestJSON-2041619958 tempest-ServerAddressesNegativeTestJSON-2041619958-project-member] [instance: cec498c8-a45a-422d-9bde-45b317a0c05a] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 721.047491] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Expecting reply to msg 16fe429ff31c4a13b9e69d0954cbcc69 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 721.190823] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 16fe429ff31c4a13b9e69d0954cbcc69 [ 721.549028] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 721.571079] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 721.571391] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 721.571535] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 721.626873] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 721.627093] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 721.627253] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 721.627643] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 721.627828] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 721.627982] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 721.637407] nova-conductor[54393]: DEBUG nova.quota [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Getting quotas for project 94445eb0173f4436a97e542b3e4ab25c. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 721.640431] nova-conductor[54393]: DEBUG nova.quota [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Getting quotas for user 0749e213ae08467b8c05b62ff1a5bde4 and project 94445eb0173f4436a97e542b3e4ab25c. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 721.645139] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] [instance: 2d3d0933-71ed-4002-a643-e3d3e9251d5d] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 721.645556] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 721.645785] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 721.646845] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 721.650792] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] [instance: 2d3d0933-71ed-4002-a643-e3d3e9251d5d] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 721.651455] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 721.651655] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 721.651823] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 721.669689] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 721.670005] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 721.670245] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 723.888294] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] [instance: e3aedb85-c405-430e-882d-7a5d4be22616] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 2088b9a7-74ef-42d6-bf31-02dece3044c4, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance e3aedb85-c405-430e-882d-7a5d4be22616 was re-scheduled: Binding failed for port 2088b9a7-74ef-42d6-bf31-02dece3044c4, please check neutron logs for more information.\n'] [ 723.888294] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 723.888294] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e3aedb85-c405-430e-882d-7a5d4be22616.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e3aedb85-c405-430e-882d-7a5d4be22616. [ 723.892716] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] [instance: e3aedb85-c405-430e-882d-7a5d4be22616] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e3aedb85-c405-430e-882d-7a5d4be22616. [ 723.919184] nova-conductor[54394]: DEBUG nova.network.neutron [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] [instance: e3aedb85-c405-430e-882d-7a5d4be22616] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 723.934368] nova-conductor[54394]: DEBUG nova.network.neutron [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] [instance: e3aedb85-c405-430e-882d-7a5d4be22616] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 723.937177] nova-conductor[54394]: DEBUG nova.network.neutron [None req-d02260dc-a43b-4c36-86f7-a16d577a67a1 tempest-ServersTestJSON-1038025062 tempest-ServersTestJSON-1038025062-project-member] [instance: e3aedb85-c405-430e-882d-7a5d4be22616] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 725.749106] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] [instance: 155db01a-6594-4e26-9bdd-f46d783fc27b] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port fb0ae627-594d-496e-8694-dfd3fdcc51ea, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 155db01a-6594-4e26-9bdd-f46d783fc27b was re-scheduled: Binding failed for port fb0ae627-594d-496e-8694-dfd3fdcc51ea, please check neutron logs for more information.\n'] [ 725.749721] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 725.749990] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 155db01a-6594-4e26-9bdd-f46d783fc27b.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 155db01a-6594-4e26-9bdd-f46d783fc27b. [ 725.750268] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] [instance: 155db01a-6594-4e26-9bdd-f46d783fc27b] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 155db01a-6594-4e26-9bdd-f46d783fc27b. [ 725.773942] nova-conductor[54393]: DEBUG nova.network.neutron [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] [instance: 155db01a-6594-4e26-9bdd-f46d783fc27b] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 725.789401] nova-conductor[54393]: DEBUG nova.network.neutron [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] [instance: 155db01a-6594-4e26-9bdd-f46d783fc27b] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 725.793352] nova-conductor[54393]: DEBUG nova.network.neutron [None req-81937dc6-729f-4132-946f-7b47a83bcd7e tempest-ImagesOneServerTestJSON-1449563660 tempest-ImagesOneServerTestJSON-1449563660-project-member] [instance: 155db01a-6594-4e26-9bdd-f46d783fc27b] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 726.115369] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Expecting reply to msg 230e5467734444188f7aef0b4f395280 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 726.278148] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 230e5467734444188f7aef0b4f395280 [ 726.619067] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 726.632263] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 726.632485] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 726.632650] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 726.710166] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 726.710382] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 726.710540] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 726.710915] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 726.711065] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 726.711219] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 726.723599] nova-conductor[54394]: DEBUG nova.quota [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Getting quotas for project ac8618fb49594cebbbbb5dbd6969a4a3. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 726.729853] nova-conductor[54394]: DEBUG nova.quota [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Getting quotas for user b9aba9a68af34d3fb1466c85dfcb6940 and project ac8618fb49594cebbbbb5dbd6969a4a3. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 726.739206] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: b8fa49d2-285a-4c75-9816-e2d2adfd98f4] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 726.739670] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 726.739871] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 726.740054] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 726.744833] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: b8fa49d2-285a-4c75-9816-e2d2adfd98f4] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 726.745489] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 726.745680] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 726.745838] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 726.773459] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 726.775001] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 726.775210] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.002s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 728.549923] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Expecting reply to msg 1c00e84944c54ebc867688772db23d82 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 728.705244] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1c00e84944c54ebc867688772db23d82 [ 729.061389] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 729.082077] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 729.082433] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 729.082700] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 729.129098] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 729.129489] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 729.129736] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 729.130383] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 729.130643] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 729.130873] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 729.142584] nova-conductor[54394]: DEBUG nova.quota [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Getting quotas for project 29bb98d6a8ce4ce0a52e42275b0325a4. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 729.146892] nova-conductor[54394]: DEBUG nova.quota [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Getting quotas for user 46306a754a734378a2398b0d7a66b0f1 and project 29bb98d6a8ce4ce0a52e42275b0325a4. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 729.162990] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] [instance: 787db05f-068c-4475-8eae-cd49aeae2d76] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 729.163649] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 729.163929] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 729.164224] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 729.191871] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] [instance: 787db05f-068c-4475-8eae-cd49aeae2d76] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 729.191871] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 729.191871] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 729.191871] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 729.261065] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 729.261499] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 729.261881] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e59826ab-3977-4faa-8e10-03a1f1c67b62 tempest-ServersAaction247Test-1765745031 tempest-ServersAaction247Test-1765745031-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 730.251201] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] [instance: 17b1a6d1-3ae0-4b0d-ba01-ef98a355a04b] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port f5438db5-d2d3-493a-8683-e3f113d8bd06, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 17b1a6d1-3ae0-4b0d-ba01-ef98a355a04b was re-scheduled: Binding failed for port f5438db5-d2d3-493a-8683-e3f113d8bd06, please check neutron logs for more information.\n'] [ 730.251800] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 730.252124] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 17b1a6d1-3ae0-4b0d-ba01-ef98a355a04b.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 17b1a6d1-3ae0-4b0d-ba01-ef98a355a04b. [ 730.252439] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] [instance: 17b1a6d1-3ae0-4b0d-ba01-ef98a355a04b] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 17b1a6d1-3ae0-4b0d-ba01-ef98a355a04b. [ 730.278131] nova-conductor[54394]: DEBUG nova.network.neutron [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] [instance: 17b1a6d1-3ae0-4b0d-ba01-ef98a355a04b] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 730.314116] nova-conductor[54394]: DEBUG nova.network.neutron [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] [instance: 17b1a6d1-3ae0-4b0d-ba01-ef98a355a04b] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 730.325750] nova-conductor[54394]: DEBUG nova.network.neutron [None req-283a7307-0b53-4530-bfbc-cc1680b54087 tempest-ServerMetadataNegativeTestJSON-1401668536 tempest-ServerMetadataNegativeTestJSON-1401668536-project-member] [instance: 17b1a6d1-3ae0-4b0d-ba01-ef98a355a04b] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 731.632270] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell0:SELECT=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 731.653796] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=3,nova_cell1:INSERT=7 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 731.664908] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=6 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 731.669743] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=6,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 732.855044] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Expecting reply to msg 55b1da283cfd46e380ab68ace51579be in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 733.063093] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 55b1da283cfd46e380ab68ace51579be [ 733.277694] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-0aa34e64-5545-4cf2-adf0-8d9686e36d99 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Targeting cell a4379ae1-47d8-4924-9d8e-3d761d765452(cell1) for conductor method rebuild_instance {{(pid=54394) wrapper /opt/stack/nova/nova/conductor/manager.py:95}} [ 733.277913] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-0aa34e64-5545-4cf2-adf0-8d9686e36d99 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 733.278308] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-0aa34e64-5545-4cf2-adf0-8d9686e36d99 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 733.278541] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-0aa34e64-5545-4cf2-adf0-8d9686e36d99 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 733.299777] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-0aa34e64-5545-4cf2-adf0-8d9686e36d99 tempest-ServerShowV254Test-1845691372 tempest-ServerShowV254Test-1845691372-project-member] [instance: c61c3592-c849-4a92-ac58-1455b4350ae3] No migration record for the rebuild/evacuate request. {{(pid=54394) rebuild_instance /opt/stack/nova/nova/conductor/manager.py:1228}} [ 733.359557] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Took 0.51 seconds to select destinations for 2 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 733.375264] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 733.375524] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 733.375697] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 733.441018] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 733.441301] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 733.441502] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 733.464106] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 733.464326] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 733.464540] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 733.464879] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 733.465055] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 733.465208] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 733.475325] nova-conductor[54394]: DEBUG nova.quota [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Getting quotas for project 676261f2506c45fb8e91c2fcb20d607a. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 733.479082] nova-conductor[54394]: DEBUG nova.quota [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Getting quotas for user e1ba0ba14f1243b78a53e164bdaa4f5e and project 676261f2506c45fb8e91c2fcb20d607a. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 733.488784] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 1cce9188-f4b8-4b96-afb4-a03dc941bd81] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 733.489246] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 733.489453] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 733.489745] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 733.494400] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 1cce9188-f4b8-4b96-afb4-a03dc941bd81] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 733.495088] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 733.495297] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 733.495461] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 733.513627] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 733.513742] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 733.513968] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 733.522936] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 07c29f21-47c3-468e-bb9e-0793306644bd] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 733.525312] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 733.525312] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 733.525312] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 733.528901] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 07c29f21-47c3-468e-bb9e-0793306644bd] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 733.529825] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 733.530129] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 733.530391] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 733.547114] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 733.547447] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 733.547726] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 734.479097] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Expecting reply to msg e981530a3c0f41a582f1a36b213f49ef in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 734.635948] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e981530a3c0f41a582f1a36b213f49ef [ 734.744765] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Expecting reply to msg 363ed08beb204b9fbcb5660560b0d384 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 734.872024] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 363ed08beb204b9fbcb5660560b0d384 [ 734.981112] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 734.993314] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 734.993542] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 734.993709] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.023631] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.023903] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.024145] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.024516] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.024733] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.024891] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.033054] nova-conductor[54394]: DEBUG nova.quota [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Getting quotas for project 6ecce658f3bc4215be540b0ce0c9f3a0. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 735.035233] nova-conductor[54394]: DEBUG nova.quota [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Getting quotas for user cdebe88552e5448c83cc709813729ce7 and project 6ecce658f3bc4215be540b0ce0c9f3a0. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 735.040884] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] [instance: 64a5a4c0-a53b-4273-844c-546789c75f03] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 735.041397] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.041610] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.041804] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.047938] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] [instance: 64a5a4c0-a53b-4273-844c-546789c75f03] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 735.048839] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.049070] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.049256] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.067240] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.067459] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.067784] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-3d1e9a0e-05bd-403c-95d6-39cea4aeaf42 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.247180] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 735.265410] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.265599] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.265771] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.286833] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.287057] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.287226] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.287594] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.287784] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.287930] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.295029] nova-conductor[54393]: DEBUG nova.quota [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Getting quotas for project 6ecce658f3bc4215be540b0ce0c9f3a0. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 735.297056] nova-conductor[54393]: DEBUG nova.quota [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Getting quotas for user cdebe88552e5448c83cc709813729ce7 and project 6ecce658f3bc4215be540b0ce0c9f3a0. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 735.301677] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] [instance: 3d1205d3-168d-423c-9384-de187e6128dc] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 735.302073] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.302268] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.302481] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.305365] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] [instance: 3d1205d3-168d-423c-9384-de187e6128dc] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 735.305965] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.306160] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.306324] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.320217] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.320624] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.320819] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4c35cbb6-b32d-4af0-be5a-e1e3ec4a6ba5 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 736.902704] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] [instance: ef5c2b13-e9d6-48a2-b56f-8b84eb62a6eb] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port e7ffb0ea-4065-4268-9ec1-71a8536dd552, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance ef5c2b13-e9d6-48a2-b56f-8b84eb62a6eb was re-scheduled: Binding failed for port e7ffb0ea-4065-4268-9ec1-71a8536dd552, please check neutron logs for more information.\n'] [ 736.902704] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 736.903051] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ef5c2b13-e9d6-48a2-b56f-8b84eb62a6eb.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ef5c2b13-e9d6-48a2-b56f-8b84eb62a6eb. [ 736.903134] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] [instance: ef5c2b13-e9d6-48a2-b56f-8b84eb62a6eb] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ef5c2b13-e9d6-48a2-b56f-8b84eb62a6eb. [ 736.929293] nova-conductor[54394]: DEBUG nova.network.neutron [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] [instance: ef5c2b13-e9d6-48a2-b56f-8b84eb62a6eb] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 736.958078] nova-conductor[54394]: DEBUG nova.network.neutron [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] [instance: ef5c2b13-e9d6-48a2-b56f-8b84eb62a6eb] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 736.961994] nova-conductor[54394]: DEBUG nova.network.neutron [None req-e6a6e908-fe2f-4568-9dd9-dd2367f39d02 tempest-ServersTestBootFromVolume-1689960611 tempest-ServersTestBootFromVolume-1689960611-project-member] [instance: ef5c2b13-e9d6-48a2-b56f-8b84eb62a6eb] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 743.589209] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] [instance: 76913c20-992d-4233-b596-3ffc16533f1c] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port f24238ce-fce8-4dd7-b719-f06d8a63b4af, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 76913c20-992d-4233-b596-3ffc16533f1c was re-scheduled: Binding failed for port f24238ce-fce8-4dd7-b719-f06d8a63b4af, please check neutron logs for more information.\n'] [ 743.590173] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 743.590463] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 76913c20-992d-4233-b596-3ffc16533f1c.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 76913c20-992d-4233-b596-3ffc16533f1c. [ 743.590728] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] [instance: 76913c20-992d-4233-b596-3ffc16533f1c] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 76913c20-992d-4233-b596-3ffc16533f1c. [ 743.616381] nova-conductor[54394]: DEBUG nova.network.neutron [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] [instance: 76913c20-992d-4233-b596-3ffc16533f1c] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 743.636382] nova-conductor[54394]: DEBUG nova.network.neutron [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] [instance: 76913c20-992d-4233-b596-3ffc16533f1c] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 743.640795] nova-conductor[54394]: DEBUG nova.network.neutron [None req-420ac789-d1c8-4655-9194-624a183cd11b tempest-MigrationsAdminTest-453482012 tempest-MigrationsAdminTest-453482012-project-member] [instance: 76913c20-992d-4233-b596-3ffc16533f1c] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 745.029819] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell0:SELECT=4 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 745.052431] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=18,nova_cell1:SAVEPOINT=4,nova_cell1:INSERT=40,nova_cell1:RELEASE=4,nova_cell1:UPDATE=3 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 745.062768] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=30,nova_api:UPDATE=3,nova_api:DELETE=2 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 745.064941] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=24,nova_api:UPDATE=2,nova_api:DELETE=3 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 745.292825] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell0:SELECT=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 745.308434] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=3,nova_cell1:INSERT=7 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 745.321452] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=6,nova_api:UPDATE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 745.321988] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=6,nova_api:DELETE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 752.086102] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Expecting reply to msg 8f4525a918454a61a51b01a285803657 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 752.361905] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8f4525a918454a61a51b01a285803657 [ 752.589214] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Took 0.51 seconds to select destinations for 3 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 752.615857] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.616102] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.616281] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.680011] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.680269] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.680444] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.725836] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.726188] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.726561] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.754113] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.754113] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.754113] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.754411] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.754614] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.754785] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.764411] nova-conductor[54393]: DEBUG nova.quota [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Getting quotas for project c569e06907d04b5aaaf35ffb74208ea8. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 752.771610] nova-conductor[54393]: DEBUG nova.quota [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Getting quotas for user 75423663ff154b0c85976280ba52b4dd and project c569e06907d04b5aaaf35ffb74208ea8. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 752.777262] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: 9c0b27bb-444c-4200-9421-60e8ef0d2a85] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 752.777779] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.778108] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.778228] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.795258] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: 9c0b27bb-444c-4200-9421-60e8ef0d2a85] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 752.795909] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.796084] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.796265] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.845940] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.846171] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.846337] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.858217] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: a50eef85-c693-4ee2-9cfa-47bfa6d95e5d] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 752.858711] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.858919] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.859084] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.864791] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: a50eef85-c693-4ee2-9cfa-47bfa6d95e5d] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 752.865500] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.865725] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.865893] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.918247] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.918536] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.918813] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.931369] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: ae47fdb7-d09d-4d02-93ef-aff0b71e63eb] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 752.932052] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.932920] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.932920] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.942044] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: ae47fdb7-d09d-4d02-93ef-aff0b71e63eb] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 752.942830] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.943078] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.943318] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.984303] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.984558] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.984727] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 754.850553] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=384,nova_cell1:UPDATE=196,nova_cell1:SAVEPOINT=33,nova_cell1:RELEASE=33,nova_cell1:INSERT=27,nova_cell1:DELETE=2 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 757.445867] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] [instance: 89363983-217b-4137-b46e-fd53ebcde670] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port afdc9a92-e1d1-4de2-ba1f-d384c094e049, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 89363983-217b-4137-b46e-fd53ebcde670 was re-scheduled: Binding failed for port afdc9a92-e1d1-4de2-ba1f-d384c094e049, please check neutron logs for more information.\n'] [ 757.446386] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 757.446681] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 89363983-217b-4137-b46e-fd53ebcde670.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 89363983-217b-4137-b46e-fd53ebcde670. [ 757.446905] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] [instance: 89363983-217b-4137-b46e-fd53ebcde670] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 89363983-217b-4137-b46e-fd53ebcde670. [ 757.500413] nova-conductor[54394]: DEBUG nova.network.neutron [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] [instance: 89363983-217b-4137-b46e-fd53ebcde670] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 757.519572] nova-conductor[54394]: DEBUG nova.network.neutron [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] [instance: 89363983-217b-4137-b46e-fd53ebcde670] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 757.523209] nova-conductor[54394]: DEBUG nova.network.neutron [None req-5efe85ba-cb5e-4784-b95a-afd601905719 tempest-ServerActionsV293TestJSON-2085577900 tempest-ServerActionsV293TestJSON-2085577900-project-member] [instance: 89363983-217b-4137-b46e-fd53ebcde670] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 759.391802] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 4c8b8d30-b619-4f48-90df-6343923b14ba] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 6285cc8d-ee1d-4cc1-9741-74e05dbaff41, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 4c8b8d30-b619-4f48-90df-6343923b14ba was re-scheduled: Binding failed for port 6285cc8d-ee1d-4cc1-9741-74e05dbaff41, please check neutron logs for more information.\n'] [ 759.392397] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 759.392622] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 4c8b8d30-b619-4f48-90df-6343923b14ba.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 4c8b8d30-b619-4f48-90df-6343923b14ba. [ 759.392836] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 4c8b8d30-b619-4f48-90df-6343923b14ba] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 4c8b8d30-b619-4f48-90df-6343923b14ba. [ 759.423871] nova-conductor[54394]: DEBUG nova.network.neutron [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 4c8b8d30-b619-4f48-90df-6343923b14ba] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 759.452026] nova-conductor[54394]: DEBUG nova.network.neutron [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 4c8b8d30-b619-4f48-90df-6343923b14ba] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 759.457349] nova-conductor[54394]: DEBUG nova.network.neutron [None req-6fc4c431-41d0-414b-a72a-bbe0423140bf tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 4c8b8d30-b619-4f48-90df-6343923b14ba] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 759.593363] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=404,nova_cell1:UPDATE=200,nova_cell1:INSERT=24,nova_cell1:SAVEPOINT=44,nova_cell1:RELEASE=44 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 759.690815] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 760.902642] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Expecting reply to msg 6b9127f3320c49d59524fd4af37e508c in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 761.053717] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6b9127f3320c49d59524fd4af37e508c [ 761.406386] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 761.418128] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 761.418339] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 761.418507] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 761.448019] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 761.448245] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 761.448413] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 761.448778] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 761.448963] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 761.449123] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 761.457350] nova-conductor[54393]: DEBUG nova.quota [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Getting quotas for project 296bd969b43b48da85debf428391be90. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 761.460371] nova-conductor[54393]: DEBUG nova.quota [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Getting quotas for user a2b21ec4d62b451c85db372f6b9ff331 and project 296bd969b43b48da85debf428391be90. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 761.465136] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 2eb583d3-43ce-4d6e-99a7-4d527cacd743] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 761.465575] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 761.465773] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 761.465940] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 761.469346] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 2eb583d3-43ce-4d6e-99a7-4d527cacd743] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 761.469941] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 761.470135] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 761.470298] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 761.485526] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 761.485728] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 761.485895] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 763.223136] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: 45450786-0b10-40b9-8564-1bd4d417f0d6] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port d97698fb-4006-427d-9b96-4b9958232288, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 45450786-0b10-40b9-8564-1bd4d417f0d6 was re-scheduled: Binding failed for port d97698fb-4006-427d-9b96-4b9958232288, please check neutron logs for more information.\n'] [ 763.223763] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 763.223997] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 45450786-0b10-40b9-8564-1bd4d417f0d6.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 45450786-0b10-40b9-8564-1bd4d417f0d6. [ 763.224331] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: 45450786-0b10-40b9-8564-1bd4d417f0d6] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 45450786-0b10-40b9-8564-1bd4d417f0d6. [ 763.246853] nova-conductor[54393]: DEBUG nova.network.neutron [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: 45450786-0b10-40b9-8564-1bd4d417f0d6] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 763.264243] nova-conductor[54393]: DEBUG nova.network.neutron [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: 45450786-0b10-40b9-8564-1bd4d417f0d6] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 763.267084] nova-conductor[54393]: DEBUG nova.network.neutron [None req-3f706d1b-5b85-44dc-8303-0a047c5ab4aa tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: 45450786-0b10-40b9-8564-1bd4d417f0d6] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 765.552268] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: d9770d22-0b9e-4d18-bcec-68cc2b491480] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port fefc0c41-3eb5-4fc9-b5c3-dd0cff86e1c5, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance d9770d22-0b9e-4d18-bcec-68cc2b491480 was re-scheduled: Binding failed for port fefc0c41-3eb5-4fc9-b5c3-dd0cff86e1c5, please check neutron logs for more information.\n'] [ 765.552844] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 765.553074] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d9770d22-0b9e-4d18-bcec-68cc2b491480.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d9770d22-0b9e-4d18-bcec-68cc2b491480. [ 765.553392] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: d9770d22-0b9e-4d18-bcec-68cc2b491480] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d9770d22-0b9e-4d18-bcec-68cc2b491480. [ 765.570566] nova-conductor[54393]: DEBUG nova.network.neutron [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: d9770d22-0b9e-4d18-bcec-68cc2b491480] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 765.584659] nova-conductor[54393]: DEBUG nova.network.neutron [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: d9770d22-0b9e-4d18-bcec-68cc2b491480] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 765.587178] nova-conductor[54393]: DEBUG nova.network.neutron [None req-f0cacced-821b-4e0f-b6bf-952f628c8e8d tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: d9770d22-0b9e-4d18-bcec-68cc2b491480] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 771.452684] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell0:SELECT=2 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 771.471812] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=10,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=29,nova_cell1:RELEASE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 771.483667] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=17,nova_api:UPDATE=1,nova_api:DELETE=3 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 771.483667] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=15,nova_api:UPDATE=3,nova_api:DELETE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 772.198295] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Expecting reply to msg c98e2afb42804bd09a4b7156f7838f7d in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 772.321982] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c98e2afb42804bd09a4b7156f7838f7d [ 772.640145] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: 5db8dac7-6f97-4684-a147-bafee0e180e8] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port b07ee759-5507-4330-982d-7b77a94495d3, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 5db8dac7-6f97-4684-a147-bafee0e180e8 was re-scheduled: Binding failed for port b07ee759-5507-4330-982d-7b77a94495d3, please check neutron logs for more information.\n'] [ 772.640733] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 772.640962] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 5db8dac7-6f97-4684-a147-bafee0e180e8.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 5db8dac7-6f97-4684-a147-bafee0e180e8. [ 772.641175] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: 5db8dac7-6f97-4684-a147-bafee0e180e8] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 5db8dac7-6f97-4684-a147-bafee0e180e8. [ 772.665126] nova-conductor[54394]: DEBUG nova.network.neutron [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: 5db8dac7-6f97-4684-a147-bafee0e180e8] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 772.684430] nova-conductor[54394]: DEBUG nova.network.neutron [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: 5db8dac7-6f97-4684-a147-bafee0e180e8] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 772.687283] nova-conductor[54394]: DEBUG nova.network.neutron [None req-eebdb233-30b0-44a6-8380-3c48546f4c0e tempest-ListServerFiltersTestJSON-1089198272 tempest-ListServerFiltersTestJSON-1089198272-project-member] [instance: 5db8dac7-6f97-4684-a147-bafee0e180e8] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 772.701685] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 772.713557] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 772.713808] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 772.713992] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 772.744698] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 772.744953] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 772.745122] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 772.745492] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 772.745667] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 772.745817] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 772.755014] nova-conductor[54394]: DEBUG nova.quota [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Getting quotas for project d10ee703bf3c4affa5b72621c111f180. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 772.758125] nova-conductor[54394]: DEBUG nova.quota [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Getting quotas for user 27e9284d43df4bd097c7b62f603c84ad and project d10ee703bf3c4affa5b72621c111f180. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 772.763009] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: 8b97820a-ed9b-488b-bb08-7839df217665] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 772.763425] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 772.763616] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 772.763775] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 772.766739] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: 8b97820a-ed9b-488b-bb08-7839df217665] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 772.767415] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 772.767606] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 772.767783] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 772.782300] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 772.782519] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 772.782715] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 774.561391] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] [instance: 1a1954da-5b61-42ea-8474-4c3ea1d6d31d] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 86becaae-9a67-4954-8876-5242b80ca86c, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 1a1954da-5b61-42ea-8474-4c3ea1d6d31d was re-scheduled: Binding failed for port 86becaae-9a67-4954-8876-5242b80ca86c, please check neutron logs for more information.\n'] [ 774.562019] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 774.562350] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1a1954da-5b61-42ea-8474-4c3ea1d6d31d.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1a1954da-5b61-42ea-8474-4c3ea1d6d31d. [ 774.563001] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] [instance: 1a1954da-5b61-42ea-8474-4c3ea1d6d31d] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1a1954da-5b61-42ea-8474-4c3ea1d6d31d. [ 774.590498] nova-conductor[54394]: DEBUG nova.network.neutron [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] [instance: 1a1954da-5b61-42ea-8474-4c3ea1d6d31d] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 774.605000] nova-conductor[54394]: DEBUG nova.network.neutron [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] [instance: 1a1954da-5b61-42ea-8474-4c3ea1d6d31d] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 774.608756] nova-conductor[54394]: DEBUG nova.network.neutron [None req-65980c5c-9c88-4fca-aaa4-2acfe6ce4d19 tempest-ServerMetadataTestJSON-2087437861 tempest-ServerMetadataTestJSON-2087437861-project-member] [instance: 1a1954da-5b61-42ea-8474-4c3ea1d6d31d] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 776.419515] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] [instance: 3940fe7c-561b-4ee7-9e36-9aa24d400a38] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port d884256a-2731-4e0a-a431-2e6123973f4c, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 3940fe7c-561b-4ee7-9e36-9aa24d400a38 was re-scheduled: Binding failed for port d884256a-2731-4e0a-a431-2e6123973f4c, please check neutron logs for more information.\n'] [ 776.420036] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 776.420287] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 3940fe7c-561b-4ee7-9e36-9aa24d400a38.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 3940fe7c-561b-4ee7-9e36-9aa24d400a38. [ 776.420507] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] [instance: 3940fe7c-561b-4ee7-9e36-9aa24d400a38] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 3940fe7c-561b-4ee7-9e36-9aa24d400a38. [ 776.464723] nova-conductor[54393]: DEBUG nova.network.neutron [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] [instance: 3940fe7c-561b-4ee7-9e36-9aa24d400a38] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 776.488418] nova-conductor[54393]: DEBUG nova.network.neutron [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] [instance: 3940fe7c-561b-4ee7-9e36-9aa24d400a38] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 776.492903] nova-conductor[54393]: DEBUG nova.network.neutron [None req-35abaa1b-6d38-443c-b632-2ad3e19e2ea5 tempest-ServerGroupTestJSON-2000420919 tempest-ServerGroupTestJSON-2000420919-project-member] [instance: 3940fe7c-561b-4ee7-9e36-9aa24d400a38] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 778.516776] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] [instance: d3ea9b40-256d-4e90-8d13-49dc08766d00] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port eabda162-5b01-4c10-8167-9edb03ed4e06, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance d3ea9b40-256d-4e90-8d13-49dc08766d00 was re-scheduled: Binding failed for port eabda162-5b01-4c10-8167-9edb03ed4e06, please check neutron logs for more information.\n'] [ 778.517355] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 778.517633] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d3ea9b40-256d-4e90-8d13-49dc08766d00.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d3ea9b40-256d-4e90-8d13-49dc08766d00. [ 778.517844] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] [instance: d3ea9b40-256d-4e90-8d13-49dc08766d00] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d3ea9b40-256d-4e90-8d13-49dc08766d00. [ 778.541512] nova-conductor[54394]: DEBUG nova.network.neutron [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] [instance: d3ea9b40-256d-4e90-8d13-49dc08766d00] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 778.588436] nova-conductor[54394]: DEBUG nova.network.neutron [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] [instance: d3ea9b40-256d-4e90-8d13-49dc08766d00] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 778.592542] nova-conductor[54394]: DEBUG nova.network.neutron [None req-9c474a7d-745e-464c-b8b5-0b952e454a9e tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] [instance: d3ea9b40-256d-4e90-8d13-49dc08766d00] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 779.591099] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Expecting reply to msg f95ac053fdaa420a92981c8123637bde in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 779.761082] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f95ac053fdaa420a92981c8123637bde [ 780.095479] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 780.108413] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 780.108650] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 780.108824] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 780.160792] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 780.161025] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 780.161191] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 780.161840] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 780.162061] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 780.162214] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 780.172711] nova-conductor[54393]: DEBUG nova.quota [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Getting quotas for project 360280d229434a2db552fe43deb7a2bb. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 780.176389] nova-conductor[54393]: DEBUG nova.quota [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Getting quotas for user 464620c67a01469ba6a526eb6555d980 and project 360280d229434a2db552fe43deb7a2bb. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 780.181298] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] [instance: 9a275012-75a8-472f-b859-b2f2cfac51c5] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 780.181984] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 780.182183] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 780.183079] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 780.187513] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] [instance: 9a275012-75a8-472f-b859-b2f2cfac51c5] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 780.188296] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 780.188501] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 780.188665] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 780.230828] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 780.230828] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 780.231029] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 781.613428] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Expecting reply to msg a2abd87919b1438bbd72cf55a0da60f3 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 781.762534] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a2abd87919b1438bbd72cf55a0da60f3 [ 781.930113] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Expecting reply to msg afb4bd04695b4d7d93d033016fd8f65a in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 782.054872] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg afb4bd04695b4d7d93d033016fd8f65a [ 782.116109] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 782.138665] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 782.138665] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 782.138665] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 782.172781] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 782.173008] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 782.173171] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 782.173546] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 782.173718] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 782.173867] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 782.182352] nova-conductor[54393]: DEBUG nova.quota [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Getting quotas for project 6e44b312d63b49dab26afc0f99c64670. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 782.184395] nova-conductor[54393]: DEBUG nova.quota [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Getting quotas for user 3c1b1c8e2f424f8194887a7542ebaaab and project 6e44b312d63b49dab26afc0f99c64670. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 782.189574] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] [instance: d1fcb7b5-ca23-4be0-b8fe-8f3b3042820e] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 782.190147] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 782.190239] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 782.190400] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 782.195626] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] [instance: d1fcb7b5-ca23-4be0-b8fe-8f3b3042820e] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 782.196714] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 782.196922] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 782.197085] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 782.212709] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 782.212913] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 782.213245] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 782.432729] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 782.445218] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 782.445732] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 782.445967] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 782.495617] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 782.495844] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 782.496034] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 782.496439] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 782.496660] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 782.496818] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 782.505455] nova-conductor[54393]: DEBUG nova.quota [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Getting quotas for project cdaf5fdfb3d5461cb787538ff6b0962a. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 782.509009] nova-conductor[54393]: DEBUG nova.quota [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Getting quotas for user a6ca13e3d6e844d4a70cb2c740a689e8 and project cdaf5fdfb3d5461cb787538ff6b0962a. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 782.518007] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] [instance: 6a01495b-b49a-46d8-9d34-52fcb27e2d2e] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 782.518007] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 782.518007] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 782.518169] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 782.523588] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] [instance: 6a01495b-b49a-46d8-9d34-52fcb27e2d2e] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 782.524471] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 782.524735] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 782.524955] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 782.548104] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 782.548346] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 782.548511] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 782.750361] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell0:SELECT=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 782.770061] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=3,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=8,nova_cell1:RELEASE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 782.778126] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=7 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 782.780132] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=5,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 792.500212] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell0:SELECT=3 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 792.527198] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=9,nova_cell1:SAVEPOINT=2,nova_cell1:INSERT=23,nova_cell1:RELEASE=2 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 792.539216] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=18,nova_api:DELETE=1,nova_api:UPDATE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 792.541279] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=18,nova_api:UPDATE=2,nova_api:DELETE=2 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 794.536512] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] [instance: 74b49da4-a792-42fa-9f01-19c17e898786] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 843aae70-005c-482c-bf41-326656197411, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 74b49da4-a792-42fa-9f01-19c17e898786 was re-scheduled: Binding failed for port 843aae70-005c-482c-bf41-326656197411, please check neutron logs for more information.\n'] [ 794.537119] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 794.537349] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 74b49da4-a792-42fa-9f01-19c17e898786.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 74b49da4-a792-42fa-9f01-19c17e898786. [ 794.537670] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] [instance: 74b49da4-a792-42fa-9f01-19c17e898786] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 74b49da4-a792-42fa-9f01-19c17e898786. [ 794.565623] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] [instance: 74b49da4-a792-42fa-9f01-19c17e898786] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 794.668055] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] [instance: 74b49da4-a792-42fa-9f01-19c17e898786] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 794.671736] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d21c45aa-918d-4f1a-ae55-b0dc240fb3ee tempest-VolumesAdminNegativeTest-1369918928 tempest-VolumesAdminNegativeTest-1369918928-project-member] [instance: 74b49da4-a792-42fa-9f01-19c17e898786] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 801.145541] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] [instance: 602c12e8-b5b5-4969-93d0-1f9fb8c464ec] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 6a05df49-a472-4e28-a449-a5e14120a54d, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 602c12e8-b5b5-4969-93d0-1f9fb8c464ec was re-scheduled: Binding failed for port 6a05df49-a472-4e28-a449-a5e14120a54d, please check neutron logs for more information.\n'] [ 801.146122] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 801.146344] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 602c12e8-b5b5-4969-93d0-1f9fb8c464ec.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 602c12e8-b5b5-4969-93d0-1f9fb8c464ec. [ 801.146549] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] [instance: 602c12e8-b5b5-4969-93d0-1f9fb8c464ec] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 602c12e8-b5b5-4969-93d0-1f9fb8c464ec. [ 801.179081] nova-conductor[54394]: DEBUG nova.network.neutron [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] [instance: 602c12e8-b5b5-4969-93d0-1f9fb8c464ec] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 801.199381] nova-conductor[54394]: DEBUG nova.network.neutron [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] [instance: 602c12e8-b5b5-4969-93d0-1f9fb8c464ec] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 801.203812] nova-conductor[54394]: DEBUG nova.network.neutron [None req-2c54f895-af2e-4788-9809-ca386a3850ca tempest-SecurityGroupsTestJSON-1959377362 tempest-SecurityGroupsTestJSON-1959377362-project-member] [instance: 602c12e8-b5b5-4969-93d0-1f9fb8c464ec] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 809.263280] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Expecting reply to msg 727805e0018c4afd9b6093ff9a544b01 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 809.481147] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 727805e0018c4afd9b6093ff9a544b01 [ 809.766260] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 809.785581] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 809.786013] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 809.786324] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 809.831590] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 809.831739] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 809.831993] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 809.837583] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 809.837771] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 809.837956] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 809.849616] nova-conductor[54394]: DEBUG nova.quota [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Getting quotas for project d4977586c76f49f08e1382ece9161191. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 809.853029] nova-conductor[54394]: DEBUG nova.quota [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Getting quotas for user 357e67371a1148deaa27ea595d2f5895 and project d4977586c76f49f08e1382ece9161191. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 809.858461] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] [instance: 87f93166-7b55-4e7a-af04-318985367bed] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 809.859037] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 809.859355] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 809.859625] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 809.863227] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] [instance: 87f93166-7b55-4e7a-af04-318985367bed] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 809.864109] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 809.864582] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 809.864845] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 809.887816] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 809.888254] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 809.888531] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 812.461193] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: 30053ec6-5706-4908-831a-5af650b391b6] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 3205d20b-32cc-4bc5-8782-0fc43aa8426a, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 30053ec6-5706-4908-831a-5af650b391b6 was re-scheduled: Binding failed for port 3205d20b-32cc-4bc5-8782-0fc43aa8426a, please check neutron logs for more information.\n'] [ 812.461821] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 812.462043] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 30053ec6-5706-4908-831a-5af650b391b6.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 30053ec6-5706-4908-831a-5af650b391b6. [ 812.462340] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: 30053ec6-5706-4908-831a-5af650b391b6] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 30053ec6-5706-4908-831a-5af650b391b6. [ 812.488476] nova-conductor[54394]: DEBUG nova.network.neutron [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: 30053ec6-5706-4908-831a-5af650b391b6] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 812.533745] nova-conductor[54394]: DEBUG nova.network.neutron [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: 30053ec6-5706-4908-831a-5af650b391b6] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 812.537740] nova-conductor[54394]: DEBUG nova.network.neutron [None req-c745e17a-ac3b-480f-8938-297b825064a4 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: 30053ec6-5706-4908-831a-5af650b391b6] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 814.577037] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] [instance: ef61895f-75c8-449d-a5e8-b9b4d6a97a45] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 6a5c9468-5460-4f32-beaa-aa7427cea39c, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance ef61895f-75c8-449d-a5e8-b9b4d6a97a45 was re-scheduled: Binding failed for port 6a5c9468-5460-4f32-beaa-aa7427cea39c, please check neutron logs for more information.\n'] [ 814.577564] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 814.577817] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ef61895f-75c8-449d-a5e8-b9b4d6a97a45.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ef61895f-75c8-449d-a5e8-b9b4d6a97a45. [ 814.578034] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] [instance: ef61895f-75c8-449d-a5e8-b9b4d6a97a45] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ef61895f-75c8-449d-a5e8-b9b4d6a97a45. [ 814.603124] nova-conductor[54393]: DEBUG nova.network.neutron [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] [instance: ef61895f-75c8-449d-a5e8-b9b4d6a97a45] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 814.618027] nova-conductor[54393]: DEBUG nova.network.neutron [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] [instance: ef61895f-75c8-449d-a5e8-b9b4d6a97a45] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 814.621813] nova-conductor[54393]: DEBUG nova.network.neutron [None req-f249177a-7698-4f4f-9e7c-bb2aeaaef2d5 tempest-ServerActionsTestOtherB-1323283040 tempest-ServerActionsTestOtherB-1323283040-project-member] [instance: ef61895f-75c8-449d-a5e8-b9b4d6a97a45] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 814.834770] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Expecting reply to msg ab8ab2741834431c85f9f8b858557b63 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 814.968147] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ab8ab2741834431c85f9f8b858557b63 [ 815.079642] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SAVEPOINT=34,nova_cell1:RELEASE=34,nova_cell1:SELECT=340,nova_cell1:UPDATE=175,nova_cell1:INSERT=28,nova_cell1:DELETE=2 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 815.336799] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Took 0.50 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 815.354277] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 815.354705] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 815.355231] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 815.388418] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 815.388639] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 815.388869] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 815.389207] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 815.389391] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 815.389553] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 815.398600] nova-conductor[54394]: DEBUG nova.quota [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Getting quotas for project ad7980bce21246ccb289123f979e9039. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 815.400587] nova-conductor[54394]: DEBUG nova.quota [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Getting quotas for user a847c2a768364ec8836bea167952cec5 and project ad7980bce21246ccb289123f979e9039. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 815.405472] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: 8283dc5e-75d0-40af-a40d-462a618f4a56] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 815.405814] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 815.406009] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 815.406171] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 815.410010] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: 8283dc5e-75d0-40af-a40d-462a618f4a56] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 815.410615] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 815.410806] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 815.410970] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 815.422978] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 815.423173] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 815.423334] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 819.018669] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: e10acb93-1ed9-4c73-a84d-ac0ae213755d] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 1d3e1b61-dabc-4a67-a071-6aa418b619ff, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance e10acb93-1ed9-4c73-a84d-ac0ae213755d was re-scheduled: Binding failed for port 1d3e1b61-dabc-4a67-a071-6aa418b619ff, please check neutron logs for more information.\n'] [ 819.019763] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 819.020029] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e10acb93-1ed9-4c73-a84d-ac0ae213755d.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e10acb93-1ed9-4c73-a84d-ac0ae213755d. [ 819.021079] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: e10acb93-1ed9-4c73-a84d-ac0ae213755d] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e10acb93-1ed9-4c73-a84d-ac0ae213755d. [ 819.039650] nova-conductor[54393]: DEBUG nova.network.neutron [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: e10acb93-1ed9-4c73-a84d-ac0ae213755d] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 819.055787] nova-conductor[54393]: DEBUG nova.network.neutron [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: e10acb93-1ed9-4c73-a84d-ac0ae213755d] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 819.060199] nova-conductor[54393]: DEBUG nova.network.neutron [None req-af586186-4234-4f48-96d2-b39372c2c54e tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: e10acb93-1ed9-4c73-a84d-ac0ae213755d] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 819.851246] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=353,nova_cell1:UPDATE=172,nova_cell1:INSERT=26,nova_cell1:SAVEPOINT=32,nova_cell1:RELEASE=32,nova_cell1:DELETE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 820.230339] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Expecting reply to msg aeb4e1fc9524448a9a4f1389082dbd68 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 820.406875] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aeb4e1fc9524448a9a4f1389082dbd68 [ 820.646781] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-8b7d2b2e-c4de-4755-a45b-e39696dbb025 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Expecting reply to msg 7c15e193f03c4b61bccaa5b28cc464cf in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 820.732667] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 820.745715] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.745954] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.746119] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 820.769387] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7c15e193f03c4b61bccaa5b28cc464cf [ 820.776265] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.776496] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.776896] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 820.777283] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.777465] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.777641] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 820.788588] nova-conductor[54394]: DEBUG nova.quota [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Getting quotas for project 71180bd316ee4554a1e30c25fa3cf608. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 820.791108] nova-conductor[54394]: DEBUG nova.quota [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Getting quotas for user fb5783207d1847f581d6c9afb7fe6f3c and project 71180bd316ee4554a1e30c25fa3cf608. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 820.796951] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] [instance: 07dd8563-86a0-4b3c-bf6f-0b5694872d7c] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 820.797443] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.797658] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.797845] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 820.803171] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] [instance: 07dd8563-86a0-4b3c-bf6f-0b5694872d7c] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 820.803848] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.804068] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.804243] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 820.819376] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.819589] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.819749] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 821.091215] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Expecting reply to msg f114adab5fc643ddbd859aca3d4938ba in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 821.152810] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-8b7d2b2e-c4de-4755-a45b-e39696dbb025 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 821.163489] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-8b7d2b2e-c4de-4755-a45b-e39696dbb025 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: ec69ccdc-ffec-4f88-b220-f721d5d3d116] While scheduling instance, the build request was already deleted. {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1707}} [ 821.197223] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] [instance: 2d3d0933-71ed-4002-a643-e3d3e9251d5d] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 78026707-bcfd-498f-8bb1-31e341e5820e, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 2d3d0933-71ed-4002-a643-e3d3e9251d5d was re-scheduled: Binding failed for port 78026707-bcfd-498f-8bb1-31e341e5820e, please check neutron logs for more information.\n'] [ 821.198028] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 821.198215] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 2d3d0933-71ed-4002-a643-e3d3e9251d5d.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 2d3d0933-71ed-4002-a643-e3d3e9251d5d. [ 821.198521] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] [instance: 2d3d0933-71ed-4002-a643-e3d3e9251d5d] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 2d3d0933-71ed-4002-a643-e3d3e9251d5d. [ 821.222732] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f114adab5fc643ddbd859aca3d4938ba [ 821.225766] nova-conductor[54394]: DEBUG nova.network.neutron [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] [instance: 2d3d0933-71ed-4002-a643-e3d3e9251d5d] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 821.241361] nova-conductor[54394]: DEBUG nova.network.neutron [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] [instance: 2d3d0933-71ed-4002-a643-e3d3e9251d5d] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 821.245602] nova-conductor[54394]: DEBUG nova.network.neutron [None req-e3e60022-6961-423a-bef8-28de2e54ea05 tempest-ServersTestMultiNic-2054523757 tempest-ServersTestMultiNic-2054523757-project-member] [instance: 2d3d0933-71ed-4002-a643-e3d3e9251d5d] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 821.351398] nova-conductor[54393]: INFO nova.scheduler.client.report [None req-8b7d2b2e-c4de-4755-a45b-e39696dbb025 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Deleted allocations for instance ec69ccdc-ffec-4f88-b220-f721d5d3d116 [ 821.359224] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8b7d2b2e-c4de-4755-a45b-e39696dbb025 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 821.359493] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8b7d2b2e-c4de-4755-a45b-e39696dbb025 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 821.359686] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8b7d2b2e-c4de-4755-a45b-e39696dbb025 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 821.360106] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8b7d2b2e-c4de-4755-a45b-e39696dbb025 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 821.360325] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8b7d2b2e-c4de-4755-a45b-e39696dbb025 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 821.360500] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8b7d2b2e-c4de-4755-a45b-e39696dbb025 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 821.371203] nova-conductor[54393]: DEBUG nova.quota [None req-8b7d2b2e-c4de-4755-a45b-e39696dbb025 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Getting quotas for project 7b6177ee33c74f3a89ab5251a94038e7. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 821.374600] nova-conductor[54393]: DEBUG nova.quota [None req-8b7d2b2e-c4de-4755-a45b-e39696dbb025 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Getting quotas for user d1429ed3916c42af92467fd7255cb2ec and project 7b6177ee33c74f3a89ab5251a94038e7. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 821.593790] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 821.608314] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 821.608757] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 821.609015] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 821.641867] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 821.642169] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 821.642560] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 821.643142] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 821.643395] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 821.643623] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 821.653388] nova-conductor[54394]: DEBUG nova.quota [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Getting quotas for project 7b6177ee33c74f3a89ab5251a94038e7. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 821.656256] nova-conductor[54394]: DEBUG nova.quota [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Getting quotas for user d1429ed3916c42af92467fd7255cb2ec and project 7b6177ee33c74f3a89ab5251a94038e7. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 821.662364] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: 7fd20631-d117-4eb4-b6a2-07ca99c640ae] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 821.662889] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 821.663234] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 821.663452] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 821.667944] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: 7fd20631-d117-4eb4-b6a2-07ca99c640ae] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 821.668723] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 821.668946] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 821.669300] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 821.692326] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 821.692573] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 821.692819] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 825.452944] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: b8fa49d2-285a-4c75-9816-e2d2adfd98f4] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port a13ca8d4-48c6-4753-9765-cbc105490ffb, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance b8fa49d2-285a-4c75-9816-e2d2adfd98f4 was re-scheduled: Binding failed for port a13ca8d4-48c6-4753-9765-cbc105490ffb, please check neutron logs for more information.\n'] [ 825.453608] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 825.453851] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b8fa49d2-285a-4c75-9816-e2d2adfd98f4.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b8fa49d2-285a-4c75-9816-e2d2adfd98f4. [ 825.454164] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: b8fa49d2-285a-4c75-9816-e2d2adfd98f4] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b8fa49d2-285a-4c75-9816-e2d2adfd98f4. [ 825.473460] nova-conductor[54394]: DEBUG nova.network.neutron [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: b8fa49d2-285a-4c75-9816-e2d2adfd98f4] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 825.488960] nova-conductor[54394]: DEBUG nova.network.neutron [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: b8fa49d2-285a-4c75-9816-e2d2adfd98f4] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 825.492597] nova-conductor[54394]: DEBUG nova.network.neutron [None req-45cf4dd1-92a3-43dd-ac1b-1d1c963960ee tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: b8fa49d2-285a-4c75-9816-e2d2adfd98f4] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 826.373454] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-89a3d589-1dce-43f7-9c5e-5c6e69e9f8c6 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Expecting reply to msg c37127a0c086402bafe6194e7ef65fae in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 826.602173] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c37127a0c086402bafe6194e7ef65fae [ 826.749964] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Expecting reply to msg 6e5f7ef756cc453ab16063ebf8b08d4c in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 826.861710] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6e5f7ef756cc453ab16063ebf8b08d4c [ 826.876287] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-89a3d589-1dce-43f7-9c5e-5c6e69e9f8c6 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 826.882804] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-89a3d589-1dce-43f7-9c5e-5c6e69e9f8c6 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 74f9fe06-0131-454a-8a75-fcc184170449] While scheduling instance, the build request was already deleted. {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1707}} [ 826.903777] nova-conductor[54394]: INFO nova.scheduler.client.report [None req-89a3d589-1dce-43f7-9c5e-5c6e69e9f8c6 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Deleted allocations for instance 74f9fe06-0131-454a-8a75-fcc184170449 [ 826.908320] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-89a3d589-1dce-43f7-9c5e-5c6e69e9f8c6 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 826.908536] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-89a3d589-1dce-43f7-9c5e-5c6e69e9f8c6 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 826.908703] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-89a3d589-1dce-43f7-9c5e-5c6e69e9f8c6 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 826.909085] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-89a3d589-1dce-43f7-9c5e-5c6e69e9f8c6 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 826.909276] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-89a3d589-1dce-43f7-9c5e-5c6e69e9f8c6 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 826.909434] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-89a3d589-1dce-43f7-9c5e-5c6e69e9f8c6 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 826.925228] nova-conductor[54394]: DEBUG nova.quota [None req-89a3d589-1dce-43f7-9c5e-5c6e69e9f8c6 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Getting quotas for project ac8618fb49594cebbbbb5dbd6969a4a3. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 826.927747] nova-conductor[54394]: DEBUG nova.quota [None req-89a3d589-1dce-43f7-9c5e-5c6e69e9f8c6 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Getting quotas for user b9aba9a68af34d3fb1466c85dfcb6940 and project ac8618fb49594cebbbbb5dbd6969a4a3. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 827.245647] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 1cce9188-f4b8-4b96-afb4-a03dc941bd81] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port c5f24d4d-1b38-435f-8eda-87ff341ee251, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 1cce9188-f4b8-4b96-afb4-a03dc941bd81 was re-scheduled: Binding failed for port c5f24d4d-1b38-435f-8eda-87ff341ee251, please check neutron logs for more information.\n'] [ 827.246177] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 827.246401] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1cce9188-f4b8-4b96-afb4-a03dc941bd81.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1cce9188-f4b8-4b96-afb4-a03dc941bd81. [ 827.246611] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 1cce9188-f4b8-4b96-afb4-a03dc941bd81] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1cce9188-f4b8-4b96-afb4-a03dc941bd81. [ 827.252710] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 827.266710] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 827.266958] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 827.267129] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 827.267468] nova-conductor[54394]: DEBUG nova.network.neutron [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 1cce9188-f4b8-4b96-afb4-a03dc941bd81] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 827.286133] nova-conductor[54394]: DEBUG nova.network.neutron [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 1cce9188-f4b8-4b96-afb4-a03dc941bd81] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 827.289393] nova-conductor[54394]: DEBUG nova.network.neutron [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 1cce9188-f4b8-4b96-afb4-a03dc941bd81] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 827.322808] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 827.323371] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 827.323681] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 827.324186] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 827.324495] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 827.324838] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 827.332426] nova-conductor[54393]: DEBUG nova.quota [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Getting quotas for project ac8618fb49594cebbbbb5dbd6969a4a3. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 827.335355] nova-conductor[54393]: DEBUG nova.quota [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Getting quotas for user b9aba9a68af34d3fb1466c85dfcb6940 and project ac8618fb49594cebbbbb5dbd6969a4a3. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 827.340316] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 104f1bf1-ac8c-4d9d-bc05-92cf29eb5624] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 827.340935] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 827.341235] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 827.341512] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 827.344887] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 104f1bf1-ac8c-4d9d-bc05-92cf29eb5624] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 827.345680] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 827.346012] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 827.346296] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 827.366727] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 827.366974] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 827.367192] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 830.059565] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-39321cdf-a773-4ba7-b123-5ba95f767b51 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Targeting cell a4379ae1-47d8-4924-9d8e-3d761d765452(cell1) for conductor method rebuild_instance {{(pid=54394) wrapper /opt/stack/nova/nova/conductor/manager.py:95}} [ 830.059807] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-39321cdf-a773-4ba7-b123-5ba95f767b51 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 830.060024] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-39321cdf-a773-4ba7-b123-5ba95f767b51 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 830.060199] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-39321cdf-a773-4ba7-b123-5ba95f767b51 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 830.074521] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-39321cdf-a773-4ba7-b123-5ba95f767b51 tempest-ServerShowV247Test-1171385314 tempest-ServerShowV247Test-1171385314-project-member] [instance: 3d1205d3-168d-423c-9384-de187e6128dc] No migration record for the rebuild/evacuate request. {{(pid=54394) rebuild_instance /opt/stack/nova/nova/conductor/manager.py:1228}} [ 831.369592] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 07c29f21-47c3-468e-bb9e-0793306644bd] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 63a6e09d-6e22-426b-a822-5c8787bd3b41, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 07c29f21-47c3-468e-bb9e-0793306644bd was re-scheduled: Binding failed for port 63a6e09d-6e22-426b-a822-5c8787bd3b41, please check neutron logs for more information.\n'] [ 831.370220] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 831.370649] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 07c29f21-47c3-468e-bb9e-0793306644bd.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 07c29f21-47c3-468e-bb9e-0793306644bd. [ 831.370750] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 07c29f21-47c3-468e-bb9e-0793306644bd] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 07c29f21-47c3-468e-bb9e-0793306644bd. [ 831.390602] nova-conductor[54394]: DEBUG nova.network.neutron [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 07c29f21-47c3-468e-bb9e-0793306644bd] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 831.406913] nova-conductor[54394]: DEBUG nova.network.neutron [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 07c29f21-47c3-468e-bb9e-0793306644bd] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 831.410615] nova-conductor[54394]: DEBUG nova.network.neutron [None req-fd159fdd-6d38-48c3-9f2a-07355f5d9c5f tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 07c29f21-47c3-468e-bb9e-0793306644bd] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 832.989719] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Expecting reply to msg 87579c1545474f5ab2e034976c7e81a9 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 833.162397] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 87579c1545474f5ab2e034976c7e81a9 [ 833.492868] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Took 0.51 seconds to select destinations for 2 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 833.506445] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 833.506704] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 833.506978] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 833.536479] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 833.536741] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 833.537102] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 833.562666] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 833.562904] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 833.563123] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 833.563563] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 833.563859] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 833.564063] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 833.575211] nova-conductor[54393]: DEBUG nova.quota [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Getting quotas for project 676261f2506c45fb8e91c2fcb20d607a. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 833.578375] nova-conductor[54393]: DEBUG nova.quota [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Getting quotas for user e1ba0ba14f1243b78a53e164bdaa4f5e and project 676261f2506c45fb8e91c2fcb20d607a. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 833.583458] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 3d421285-84bf-49f1-b54e-8968ca03feba] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 833.583970] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 833.584196] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 833.584359] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 833.588856] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 3d421285-84bf-49f1-b54e-8968ca03feba] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 833.589518] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 833.589713] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 833.589871] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 833.608889] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 833.609100] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 833.609262] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 833.615633] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 2f53ec9e-c44c-4c55-bcbd-0be90249b989] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 833.616119] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 833.616346] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 833.616510] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 833.619240] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 2f53ec9e-c44c-4c55-bcbd-0be90249b989] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 833.619855] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 833.620096] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 833.620271] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 833.641789] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 833.642093] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 833.642287] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 836.913381] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell0:SELECT=5 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 840.058337] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=28,nova_api:UPDATE=3,nova_api:DELETE=4 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 840.079311] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=31,nova_api:DELETE=1,nova_api:UPDATE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 840.095218] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=17,nova_cell1:SAVEPOINT=2,nova_cell1:INSERT=31,nova_cell1:RELEASE=2,nova_cell1:UPDATE=3 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 843.568662] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell0:SELECT=3 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 843.623908] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=9,nova_cell1:INSERT=21 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 843.636559] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=20,nova_api:UPDATE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 843.641572] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=17,nova_api:DELETE=4,nova_api:UPDATE=2 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 851.254518] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: 9c0b27bb-444c-4200-9421-60e8ef0d2a85] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port be8fef03-3bfd-4994-8938-2a6f8a65934b, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 9c0b27bb-444c-4200-9421-60e8ef0d2a85 was re-scheduled: Binding failed for port be8fef03-3bfd-4994-8938-2a6f8a65934b, please check neutron logs for more information.\n'] [ 851.255127] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 851.255355] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 9c0b27bb-444c-4200-9421-60e8ef0d2a85.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 9c0b27bb-444c-4200-9421-60e8ef0d2a85. [ 851.255563] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: 9c0b27bb-444c-4200-9421-60e8ef0d2a85] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 9c0b27bb-444c-4200-9421-60e8ef0d2a85. [ 851.283840] nova-conductor[54394]: DEBUG nova.network.neutron [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: 9c0b27bb-444c-4200-9421-60e8ef0d2a85] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 851.382818] nova-conductor[54394]: DEBUG nova.network.neutron [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: 9c0b27bb-444c-4200-9421-60e8ef0d2a85] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 851.386513] nova-conductor[54394]: DEBUG nova.network.neutron [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: 9c0b27bb-444c-4200-9421-60e8ef0d2a85] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 857.642294] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: a50eef85-c693-4ee2-9cfa-47bfa6d95e5d] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 0c15e363-adf0-4038-a8ad-c60a777f7fc1, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance a50eef85-c693-4ee2-9cfa-47bfa6d95e5d was re-scheduled: Binding failed for port 0c15e363-adf0-4038-a8ad-c60a777f7fc1, please check neutron logs for more information.\n'] [ 857.642968] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 857.643210] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance a50eef85-c693-4ee2-9cfa-47bfa6d95e5d.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance a50eef85-c693-4ee2-9cfa-47bfa6d95e5d. [ 857.643531] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: a50eef85-c693-4ee2-9cfa-47bfa6d95e5d] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance a50eef85-c693-4ee2-9cfa-47bfa6d95e5d. [ 857.662939] nova-conductor[54393]: DEBUG nova.network.neutron [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: a50eef85-c693-4ee2-9cfa-47bfa6d95e5d] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 857.677693] nova-conductor[54393]: DEBUG nova.network.neutron [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: a50eef85-c693-4ee2-9cfa-47bfa6d95e5d] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 857.680784] nova-conductor[54393]: DEBUG nova.network.neutron [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: a50eef85-c693-4ee2-9cfa-47bfa6d95e5d] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 859.403465] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: ae47fdb7-d09d-4d02-93ef-aff0b71e63eb] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port ee00b29c-c7e8-46ff-88af-56c176c7cfaa, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance ae47fdb7-d09d-4d02-93ef-aff0b71e63eb was re-scheduled: Binding failed for port ee00b29c-c7e8-46ff-88af-56c176c7cfaa, please check neutron logs for more information.\n'] [ 859.404030] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 859.404248] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ae47fdb7-d09d-4d02-93ef-aff0b71e63eb.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ae47fdb7-d09d-4d02-93ef-aff0b71e63eb. [ 859.404460] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: ae47fdb7-d09d-4d02-93ef-aff0b71e63eb] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ae47fdb7-d09d-4d02-93ef-aff0b71e63eb. [ 859.421608] nova-conductor[54394]: DEBUG nova.network.neutron [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: ae47fdb7-d09d-4d02-93ef-aff0b71e63eb] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 859.435799] nova-conductor[54394]: DEBUG nova.network.neutron [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: ae47fdb7-d09d-4d02-93ef-aff0b71e63eb] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 859.438319] nova-conductor[54394]: DEBUG nova.network.neutron [None req-c6a23d6e-d285-49c9-adec-316f6af549e9 tempest-ListServersNegativeTestJSON-423547973 tempest-ListServersNegativeTestJSON-423547973-project-member] [instance: ae47fdb7-d09d-4d02-93ef-aff0b71e63eb] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 862.172394] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 2eb583d3-43ce-4d6e-99a7-4d527cacd743] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port c6a84c8c-ce5a-4e86-b551-767d0036f572, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 2eb583d3-43ce-4d6e-99a7-4d527cacd743 was re-scheduled: Binding failed for port c6a84c8c-ce5a-4e86-b551-767d0036f572, please check neutron logs for more information.\n'] [ 862.172999] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 862.173335] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 2eb583d3-43ce-4d6e-99a7-4d527cacd743.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 2eb583d3-43ce-4d6e-99a7-4d527cacd743. [ 862.173568] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 2eb583d3-43ce-4d6e-99a7-4d527cacd743] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 2eb583d3-43ce-4d6e-99a7-4d527cacd743. [ 862.214353] nova-conductor[54393]: DEBUG nova.network.neutron [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 2eb583d3-43ce-4d6e-99a7-4d527cacd743] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 862.321534] nova-conductor[54393]: DEBUG nova.network.neutron [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 2eb583d3-43ce-4d6e-99a7-4d527cacd743] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 862.325181] nova-conductor[54393]: DEBUG nova.network.neutron [None req-a7704385-b448-4ff9-8002-a69c2b071f2e tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: 2eb583d3-43ce-4d6e-99a7-4d527cacd743] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 863.514856] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Expecting reply to msg 7ea0baa72cfb43c0b85c2f96fcb1f840 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 863.639578] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7ea0baa72cfb43c0b85c2f96fcb1f840 [ 863.948947] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: 8b97820a-ed9b-488b-bb08-7839df217665] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 87335da5-1a63-4801-a9f2-58a0cd70e780, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 8b97820a-ed9b-488b-bb08-7839df217665 was re-scheduled: Binding failed for port 87335da5-1a63-4801-a9f2-58a0cd70e780, please check neutron logs for more information.\n'] [ 863.949595] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 863.949779] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8b97820a-ed9b-488b-bb08-7839df217665.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8b97820a-ed9b-488b-bb08-7839df217665. [ 863.949987] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: 8b97820a-ed9b-488b-bb08-7839df217665] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8b97820a-ed9b-488b-bb08-7839df217665. [ 863.977020] nova-conductor[54393]: DEBUG nova.network.neutron [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: 8b97820a-ed9b-488b-bb08-7839df217665] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 864.002389] nova-conductor[54393]: DEBUG nova.network.neutron [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: 8b97820a-ed9b-488b-bb08-7839df217665] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 864.009172] nova-conductor[54393]: DEBUG nova.network.neutron [None req-b02bd798-071a-4db4-b88a-3f95f9111864 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: 8b97820a-ed9b-488b-bb08-7839df217665] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 864.017844] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 864.030348] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 864.030605] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 864.031152] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 864.059866] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 864.060227] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 864.060453] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 864.060916] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 864.061170] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 864.061389] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 864.069690] nova-conductor[54394]: DEBUG nova.quota [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Getting quotas for project 296bd969b43b48da85debf428391be90. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 864.072074] nova-conductor[54394]: DEBUG nova.quota [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Getting quotas for user a2b21ec4d62b451c85db372f6b9ff331 and project 296bd969b43b48da85debf428391be90. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 864.078697] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: d517b440-7c3e-446a-827e-79dfbcd71133] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 864.079087] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 864.079484] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 864.079710] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 864.083784] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: d517b440-7c3e-446a-827e-79dfbcd71133] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 864.084816] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 864.084816] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 864.085052] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 864.115381] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 864.115855] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 864.116336] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 865.421450] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Expecting reply to msg 01847ee90cd6470080f97ebd44bebc9b in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 865.548441] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 01847ee90cd6470080f97ebd44bebc9b [ 865.923666] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Took 0.50 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 865.944999] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 865.945384] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 865.945668] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 866.020935] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 866.021114] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 866.021290] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 866.021666] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 866.021848] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 866.022008] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 866.030611] nova-conductor[54393]: DEBUG nova.quota [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Getting quotas for project 6029e7851cd540c9a645962243410d34. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 866.033885] nova-conductor[54393]: DEBUG nova.quota [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Getting quotas for user c4beffab6c2d401a9153612bb906041d and project 6029e7851cd540c9a645962243410d34. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 866.039159] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] [instance: 73d9508c-a07c-4c38-b5ab-5470ffc1d91a] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 866.039677] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 866.039898] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 866.040169] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 866.046371] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] [instance: 73d9508c-a07c-4c38-b5ab-5470ffc1d91a] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 866.047103] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 866.047245] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 866.047405] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 866.061809] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 866.062023] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 866.062192] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 866.358143] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Expecting reply to msg 8ae3ca45195c4571b254b84e708c918c in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 866.499962] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8ae3ca45195c4571b254b84e708c918c [ 866.860462] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 866.873040] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 866.873278] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 866.873446] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 866.901167] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 866.901387] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 866.901601] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 866.901967] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 866.902142] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 866.902296] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 866.910459] nova-conductor[54393]: DEBUG nova.quota [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Getting quotas for project d10ee703bf3c4affa5b72621c111f180. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 866.913409] nova-conductor[54393]: DEBUG nova.quota [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Getting quotas for user 27e9284d43df4bd097c7b62f603c84ad and project d10ee703bf3c4affa5b72621c111f180. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 866.918269] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: ca3f90c9-0ea2-4c63-8009-b875d65537e5] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 866.918732] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 866.918935] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 866.919125] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 866.923676] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: ca3f90c9-0ea2-4c63-8009-b875d65537e5] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 866.924325] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 866.924522] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 866.924687] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 866.942492] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 866.942621] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 866.942830] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 867.478892] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] [instance: 9a275012-75a8-472f-b859-b2f2cfac51c5] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port fd592e9e-8b38-4031-8bc5-b261a8115985, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 9a275012-75a8-472f-b859-b2f2cfac51c5 was re-scheduled: Binding failed for port fd592e9e-8b38-4031-8bc5-b261a8115985, please check neutron logs for more information.\n'] [ 867.479498] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 867.480498] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 9a275012-75a8-472f-b859-b2f2cfac51c5.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 9a275012-75a8-472f-b859-b2f2cfac51c5. [ 867.480498] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] [instance: 9a275012-75a8-472f-b859-b2f2cfac51c5] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 9a275012-75a8-472f-b859-b2f2cfac51c5. [ 867.506315] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] [instance: 9a275012-75a8-472f-b859-b2f2cfac51c5] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 867.520640] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] [instance: 9a275012-75a8-472f-b859-b2f2cfac51c5] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 867.523757] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d91b9239-b7d7-481f-9adb-6f62c037227e tempest-ServerActionsTestJSON-46582305 tempest-ServerActionsTestJSON-46582305-project-member] [instance: 9a275012-75a8-472f-b859-b2f2cfac51c5] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 869.290606] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] [instance: d1fcb7b5-ca23-4be0-b8fe-8f3b3042820e] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 65830efe-b0e7-4f0a-a7a0-cd5a1e6eae2e, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance d1fcb7b5-ca23-4be0-b8fe-8f3b3042820e was re-scheduled: Binding failed for port 65830efe-b0e7-4f0a-a7a0-cd5a1e6eae2e, please check neutron logs for more information.\n'] [ 869.291762] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 869.292078] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d1fcb7b5-ca23-4be0-b8fe-8f3b3042820e.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d1fcb7b5-ca23-4be0-b8fe-8f3b3042820e. [ 869.292470] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] [instance: d1fcb7b5-ca23-4be0-b8fe-8f3b3042820e] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d1fcb7b5-ca23-4be0-b8fe-8f3b3042820e. [ 869.315542] nova-conductor[54394]: DEBUG nova.network.neutron [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] [instance: d1fcb7b5-ca23-4be0-b8fe-8f3b3042820e] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 869.345673] nova-conductor[54394]: DEBUG nova.network.neutron [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] [instance: d1fcb7b5-ca23-4be0-b8fe-8f3b3042820e] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 869.348741] nova-conductor[54394]: DEBUG nova.network.neutron [None req-d78ec12d-c4d7-4e58-9072-b93954ae29c4 tempest-ServerRescueTestJSONUnderV235-1808558353 tempest-ServerRescueTestJSONUnderV235-1808558353-project-member] [instance: d1fcb7b5-ca23-4be0-b8fe-8f3b3042820e] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 870.257727] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Expecting reply to msg 7b7c61c1a5cd4508bb1716b70e7577ba in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 870.439384] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7b7c61c1a5cd4508bb1716b70e7577ba [ 870.762959] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 870.778177] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 870.778422] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 870.778592] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 870.852050] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 870.852516] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 870.852829] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 870.853372] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 870.853699] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 870.853986] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 870.891177] nova-conductor[54394]: DEBUG nova.quota [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Getting quotas for project db52452c3a154231aee48bec93803540. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 870.897138] nova-conductor[54394]: DEBUG nova.quota [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Getting quotas for user 92c546b8b8cd4a5998dcf4cea4f6c86b and project db52452c3a154231aee48bec93803540. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 870.914746] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] [instance: 24c09d77-d637-461a-8c9f-0d8e5f6e60b5] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 870.915066] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 870.915289] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 870.915455] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 870.937507] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] [instance: 24c09d77-d637-461a-8c9f-0d8e5f6e60b5] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 870.938276] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 870.938477] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 870.938638] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 870.954763] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 870.954978] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 870.955140] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-210f60e0-a579-4856-a033-d8b8d57e9521 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 871.072553] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] [instance: 6a01495b-b49a-46d8-9d34-52fcb27e2d2e] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port a7879089-24ad-4560-b914-4d5c5d2e951b, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 6a01495b-b49a-46d8-9d34-52fcb27e2d2e was re-scheduled: Binding failed for port a7879089-24ad-4560-b914-4d5c5d2e951b, please check neutron logs for more information.\n'] [ 871.073130] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 871.073361] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 6a01495b-b49a-46d8-9d34-52fcb27e2d2e.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 6a01495b-b49a-46d8-9d34-52fcb27e2d2e. [ 871.073570] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] [instance: 6a01495b-b49a-46d8-9d34-52fcb27e2d2e] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 6a01495b-b49a-46d8-9d34-52fcb27e2d2e. [ 871.094119] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] [instance: 6a01495b-b49a-46d8-9d34-52fcb27e2d2e] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 871.121406] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] [instance: 6a01495b-b49a-46d8-9d34-52fcb27e2d2e] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 871.125674] nova-conductor[54393]: DEBUG nova.network.neutron [None req-d0e96852-7fc2-4ada-9df2-25afc9d55819 tempest-AttachVolumeShelveTestJSON-460913406 tempest-AttachVolumeShelveTestJSON-460913406-project-member] [instance: 6a01495b-b49a-46d8-9d34-52fcb27e2d2e] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 875.469406] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] [instance: 87f93166-7b55-4e7a-af04-318985367bed] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 59e7a6d5-5028-41da-aba3-577908d97883, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 87f93166-7b55-4e7a-af04-318985367bed was re-scheduled: Binding failed for port 59e7a6d5-5028-41da-aba3-577908d97883, please check neutron logs for more information.\n'] [ 875.470196] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 875.470505] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 87f93166-7b55-4e7a-af04-318985367bed.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 87f93166-7b55-4e7a-af04-318985367bed. [ 875.471217] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] [instance: 87f93166-7b55-4e7a-af04-318985367bed] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 87f93166-7b55-4e7a-af04-318985367bed. [ 875.475106] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:UPDATE=193,nova_cell1:INSERT=33,nova_cell1:SELECT=388,nova_cell1:SAVEPOINT=48,nova_cell1:RELEASE=48 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 875.494664] nova-conductor[54394]: DEBUG nova.network.neutron [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] [instance: 87f93166-7b55-4e7a-af04-318985367bed] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 875.512546] nova-conductor[54394]: DEBUG nova.network.neutron [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] [instance: 87f93166-7b55-4e7a-af04-318985367bed] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 875.515717] nova-conductor[54394]: DEBUG nova.network.neutron [None req-36be1454-fcdf-42e2-b42e-4d3e0c050a81 tempest-ServersTestFqdnHostnames-1053264386 tempest-ServersTestFqdnHostnames-1053264386-project-member] [instance: 87f93166-7b55-4e7a-af04-318985367bed] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 876.905681] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell0:SELECT=2 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 876.926419] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=6,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=15,nova_cell1:RELEASE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 876.936076] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=2,nova_api:DELETE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 876.940588] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 876.995809] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: 8283dc5e-75d0-40af-a40d-462a618f4a56] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port c6a5b362-d763-43b2-90ee-5b61abf27952, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 8283dc5e-75d0-40af-a40d-462a618f4a56 was re-scheduled: Binding failed for port c6a5b362-d763-43b2-90ee-5b61abf27952, please check neutron logs for more information.\n'] [ 876.996884] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 876.997320] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8283dc5e-75d0-40af-a40d-462a618f4a56.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8283dc5e-75d0-40af-a40d-462a618f4a56. [ 876.998051] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: 8283dc5e-75d0-40af-a40d-462a618f4a56] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8283dc5e-75d0-40af-a40d-462a618f4a56. [ 877.020486] nova-conductor[54393]: DEBUG nova.network.neutron [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: 8283dc5e-75d0-40af-a40d-462a618f4a56] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 877.039061] nova-conductor[54393]: DEBUG nova.network.neutron [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: 8283dc5e-75d0-40af-a40d-462a618f4a56] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 877.048257] nova-conductor[54393]: DEBUG nova.network.neutron [None req-1e5956c6-6137-4d3e-831b-c044e6eb7966 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: 8283dc5e-75d0-40af-a40d-462a618f4a56] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 879.949233] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:UPDATE=227,nova_cell1:SELECT=426,nova_cell1:INSERT=25,nova_cell1:SAVEPOINT=32,nova_cell1:RELEASE=32,nova_cell1:DELETE=4 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 880.445615] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Expecting reply to msg 2efdc41523b848c19a36969c886711f5 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 880.593941] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2efdc41523b848c19a36969c886711f5 [ 880.858631] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell0:SELECT=2 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 880.940777] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=6,nova_cell1:INSERT=15,nova_cell1:SAVEPOINT=1,nova_cell1:RELEASE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 880.948596] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 880.960768] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 880.961000] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 880.961167] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 880.993157] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 880.993157] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 880.993157] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 880.995251] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 880.995251] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 880.995712] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 881.005131] nova-conductor[54394]: DEBUG nova.quota [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Getting quotas for project ad7980bce21246ccb289123f979e9039. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 881.008686] nova-conductor[54394]: DEBUG nova.quota [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Getting quotas for user a847c2a768364ec8836bea167952cec5 and project ad7980bce21246ccb289123f979e9039. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 881.013882] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: ece742e3-4af5-4e24-ba29-c37f77c70e25] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 881.014539] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 881.014625] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 881.014821] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 881.019224] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: ece742e3-4af5-4e24-ba29-c37f77c70e25] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 881.019982] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 881.020422] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 881.020422] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 881.038991] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 881.039260] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 881.039498] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 881.451592] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Expecting reply to msg 8f10b61ce6654736af90a60321acefee in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 881.595868] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8f10b61ce6654736af90a60321acefee [ 881.955191] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 881.969847] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 881.970691] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 881.970807] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 882.024075] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 882.024332] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 882.024502] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 882.025045] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 882.026388] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 882.026388] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 882.036904] nova-conductor[54394]: DEBUG nova.quota [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Getting quotas for project 051867b093ea4314a18cbec080618750. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 882.040344] nova-conductor[54394]: DEBUG nova.quota [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Getting quotas for user 70ba1d9ac9c94c4797d835b1db24c6f7 and project 051867b093ea4314a18cbec080618750. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 882.045600] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] [instance: c34de847-9df4-4393-b691-a689a24b9a3f] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 882.046214] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 882.046427] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 882.046596] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 882.054265] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] [instance: c34de847-9df4-4393-b691-a689a24b9a3f] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 882.055079] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 882.055341] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 882.055525] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 882.095214] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 882.095795] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 882.096129] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 892.031219] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell0:SELECT=2 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 892.062704] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=6,nova_cell1:INSERT=15,nova_cell1:SAVEPOINT=1,nova_cell1:RELEASE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 892.087588] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=25,nova_api:UPDATE=3,nova_api:DELETE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 892.089704] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=23,nova_api:DELETE=3,nova_api:UPDATE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 900.925334] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] [instance: 07dd8563-86a0-4b3c-bf6f-0b5694872d7c] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 76aa3ea6-5a05-4f04-8436-459b98c0805d, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 07dd8563-86a0-4b3c-bf6f-0b5694872d7c was re-scheduled: Binding failed for port 76aa3ea6-5a05-4f04-8436-459b98c0805d, please check neutron logs for more information.\n'] [ 900.925905] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 900.926145] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 07dd8563-86a0-4b3c-bf6f-0b5694872d7c.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 07dd8563-86a0-4b3c-bf6f-0b5694872d7c. [ 900.926467] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] [instance: 07dd8563-86a0-4b3c-bf6f-0b5694872d7c] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 07dd8563-86a0-4b3c-bf6f-0b5694872d7c. [ 900.949158] nova-conductor[54394]: DEBUG nova.network.neutron [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] [instance: 07dd8563-86a0-4b3c-bf6f-0b5694872d7c] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 900.963230] nova-conductor[54394]: DEBUG nova.network.neutron [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] [instance: 07dd8563-86a0-4b3c-bf6f-0b5694872d7c] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 900.966283] nova-conductor[54394]: DEBUG nova.network.neutron [None req-dd83eb37-00b6-4900-bbd1-acd594c2e04c tempest-ServerPasswordTestJSON-2033483251 tempest-ServerPasswordTestJSON-2033483251-project-member] [instance: 07dd8563-86a0-4b3c-bf6f-0b5694872d7c] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 904.906242] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: 7fd20631-d117-4eb4-b6a2-07ca99c640ae] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 0ee096e2-8374-4819-b88f-feb8fd5f8ee2, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 7fd20631-d117-4eb4-b6a2-07ca99c640ae was re-scheduled: Binding failed for port 0ee096e2-8374-4819-b88f-feb8fd5f8ee2, please check neutron logs for more information.\n'] [ 904.906917] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 904.907173] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7fd20631-d117-4eb4-b6a2-07ca99c640ae.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7fd20631-d117-4eb4-b6a2-07ca99c640ae. [ 904.907378] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: 7fd20631-d117-4eb4-b6a2-07ca99c640ae] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7fd20631-d117-4eb4-b6a2-07ca99c640ae. [ 904.939601] nova-conductor[54393]: DEBUG nova.network.neutron [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: 7fd20631-d117-4eb4-b6a2-07ca99c640ae] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 904.963971] nova-conductor[54393]: DEBUG nova.network.neutron [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: 7fd20631-d117-4eb4-b6a2-07ca99c640ae] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 904.968165] nova-conductor[54393]: DEBUG nova.network.neutron [None req-bf663148-9a50-4cba-8a70-b1bc2de63cc2 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: 7fd20631-d117-4eb4-b6a2-07ca99c640ae] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 905.682699] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Expecting reply to msg 263fc44ecdbd4261bb3ca7a0716a907e in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 905.809321] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 263fc44ecdbd4261bb3ca7a0716a907e [ 906.185500] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 906.198870] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 906.199147] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 906.199322] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 906.229463] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 906.229947] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 906.230261] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 906.230909] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 906.231235] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 906.231501] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 906.239280] nova-conductor[54393]: DEBUG nova.quota [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Getting quotas for project 7b6177ee33c74f3a89ab5251a94038e7. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 906.241949] nova-conductor[54393]: DEBUG nova.quota [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Getting quotas for user d1429ed3916c42af92467fd7255cb2ec and project 7b6177ee33c74f3a89ab5251a94038e7. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 906.247068] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: 98f26762-8ec1-4925-9a0a-5e25df3e7934] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 906.247684] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 906.248036] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 906.248338] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 906.254811] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: 98f26762-8ec1-4925-9a0a-5e25df3e7934] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 906.255644] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 906.255816] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 906.255977] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 906.277110] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 906.277405] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 906.277631] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 906.738531] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-79dc58c5-e0db-4448-bf34-f19fbc99b514 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Targeting cell a4379ae1-47d8-4924-9d8e-3d761d765452(cell1) for conductor method rebuild_instance {{(pid=54394) wrapper /opt/stack/nova/nova/conductor/manager.py:95}} [ 906.738806] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-79dc58c5-e0db-4448-bf34-f19fbc99b514 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 906.738952] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-79dc58c5-e0db-4448-bf34-f19fbc99b514 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 906.739110] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-79dc58c5-e0db-4448-bf34-f19fbc99b514 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 906.746561] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-79dc58c5-e0db-4448-bf34-f19fbc99b514 tempest-ServerShowV257Test-14277376 tempest-ServerShowV257Test-14277376-project-member] [instance: 24c09d77-d637-461a-8c9f-0d8e5f6e60b5] No migration record for the rebuild/evacuate request. {{(pid=54394) rebuild_instance /opt/stack/nova/nova/conductor/manager.py:1228}} [ 908.883314] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 104f1bf1-ac8c-4d9d-bc05-92cf29eb5624] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 0a72e766-0943-48e3-9979-33b44617bf3f, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 104f1bf1-ac8c-4d9d-bc05-92cf29eb5624 was re-scheduled: Binding failed for port 0a72e766-0943-48e3-9979-33b44617bf3f, please check neutron logs for more information.\n'] [ 908.883957] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 908.884220] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 104f1bf1-ac8c-4d9d-bc05-92cf29eb5624.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 104f1bf1-ac8c-4d9d-bc05-92cf29eb5624. [ 908.884530] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 104f1bf1-ac8c-4d9d-bc05-92cf29eb5624] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 104f1bf1-ac8c-4d9d-bc05-92cf29eb5624. [ 908.907895] nova-conductor[54393]: DEBUG nova.network.neutron [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 104f1bf1-ac8c-4d9d-bc05-92cf29eb5624] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 908.925048] nova-conductor[54393]: DEBUG nova.network.neutron [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 104f1bf1-ac8c-4d9d-bc05-92cf29eb5624] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 908.929309] nova-conductor[54393]: DEBUG nova.network.neutron [None req-4a306b1d-2414-423c-b634-b4a8cc2f4c1f tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 104f1bf1-ac8c-4d9d-bc05-92cf29eb5624] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 910.248798] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Expecting reply to msg 3519181c04c043e99e12ed20df362f55 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 910.384467] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3519181c04c043e99e12ed20df362f55 [ 910.587358] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 3d421285-84bf-49f1-b54e-8968ca03feba] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 2b4094e8-118f-4f8f-955f-fe0f2fc18ee1, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 3d421285-84bf-49f1-b54e-8968ca03feba was re-scheduled: Binding failed for port 2b4094e8-118f-4f8f-955f-fe0f2fc18ee1, please check neutron logs for more information.\n'] [ 910.587929] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 910.588411] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 3d421285-84bf-49f1-b54e-8968ca03feba.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 3d421285-84bf-49f1-b54e-8968ca03feba. [ 910.588639] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 3d421285-84bf-49f1-b54e-8968ca03feba] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 3d421285-84bf-49f1-b54e-8968ca03feba. [ 910.612467] nova-conductor[54393]: DEBUG nova.network.neutron [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 3d421285-84bf-49f1-b54e-8968ca03feba] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 910.627261] nova-conductor[54393]: DEBUG nova.network.neutron [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 3d421285-84bf-49f1-b54e-8968ca03feba] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 910.630839] nova-conductor[54393]: DEBUG nova.network.neutron [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 3d421285-84bf-49f1-b54e-8968ca03feba] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 910.751237] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 910.764056] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 910.764361] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 910.764571] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 910.802098] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 910.802403] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 910.802620] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 910.803097] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 910.803270] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 910.803471] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 910.812258] nova-conductor[54394]: DEBUG nova.quota [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Getting quotas for project ac8618fb49594cebbbbb5dbd6969a4a3. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 910.815480] nova-conductor[54394]: DEBUG nova.quota [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Getting quotas for user b9aba9a68af34d3fb1466c85dfcb6940 and project ac8618fb49594cebbbbb5dbd6969a4a3. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 910.820527] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: c142f8c1-93e7-4a3e-9c44-9998b9a800a2] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 910.821009] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 910.821249] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 910.821452] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 910.824293] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: c142f8c1-93e7-4a3e-9c44-9998b9a800a2] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 910.824932] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 910.825168] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 910.825375] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 910.839014] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 910.839256] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 910.839465] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 912.321315] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 2f53ec9e-c44c-4c55-bcbd-0be90249b989] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 04476da6-121f-4421-b81b-04362cf5b39b, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 2f53ec9e-c44c-4c55-bcbd-0be90249b989 was re-scheduled: Binding failed for port 04476da6-121f-4421-b81b-04362cf5b39b, please check neutron logs for more information.\n'] [ 912.321920] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 912.322153] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 2f53ec9e-c44c-4c55-bcbd-0be90249b989.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 2f53ec9e-c44c-4c55-bcbd-0be90249b989. [ 912.322456] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 2f53ec9e-c44c-4c55-bcbd-0be90249b989] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 2f53ec9e-c44c-4c55-bcbd-0be90249b989. [ 912.340233] nova-conductor[54394]: DEBUG nova.network.neutron [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 2f53ec9e-c44c-4c55-bcbd-0be90249b989] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 912.355765] nova-conductor[54394]: DEBUG nova.network.neutron [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 2f53ec9e-c44c-4c55-bcbd-0be90249b989] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 912.358386] nova-conductor[54394]: DEBUG nova.network.neutron [None req-cbace493-d20f-42ec-935c-bba852b4d450 tempest-MultipleCreateTestJSON-1503098407 tempest-MultipleCreateTestJSON-1503098407-project-member] [instance: 2f53ec9e-c44c-4c55-bcbd-0be90249b989] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 914.055367] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: d517b440-7c3e-446a-827e-79dfbcd71133] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 2f481655-3eca-4643-8b70-4ed2872b75eb, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance d517b440-7c3e-446a-827e-79dfbcd71133 was re-scheduled: Binding failed for port 2f481655-3eca-4643-8b70-4ed2872b75eb, please check neutron logs for more information.\n'] [ 914.055981] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 914.056237] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d517b440-7c3e-446a-827e-79dfbcd71133.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d517b440-7c3e-446a-827e-79dfbcd71133. [ 914.056446] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: d517b440-7c3e-446a-827e-79dfbcd71133] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d517b440-7c3e-446a-827e-79dfbcd71133. [ 914.079486] nova-conductor[54393]: DEBUG nova.network.neutron [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: d517b440-7c3e-446a-827e-79dfbcd71133] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 914.099816] nova-conductor[54393]: DEBUG nova.network.neutron [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: d517b440-7c3e-446a-827e-79dfbcd71133] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 914.102628] nova-conductor[54393]: DEBUG nova.network.neutron [None req-858da8a8-b398-4ecb-b0c0-66c84c7673e6 tempest-ImagesTestJSON-1436934721 tempest-ImagesTestJSON-1436934721-project-member] [instance: d517b440-7c3e-446a-827e-79dfbcd71133] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 915.805554] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] [instance: 73d9508c-a07c-4c38-b5ab-5470ffc1d91a] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port c515e861-f41b-4801-b858-9a49deefd2aa, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 73d9508c-a07c-4c38-b5ab-5470ffc1d91a was re-scheduled: Binding failed for port c515e861-f41b-4801-b858-9a49deefd2aa, please check neutron logs for more information.\n'] [ 915.806163] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 915.806443] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 73d9508c-a07c-4c38-b5ab-5470ffc1d91a.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 73d9508c-a07c-4c38-b5ab-5470ffc1d91a. [ 915.806632] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] [instance: 73d9508c-a07c-4c38-b5ab-5470ffc1d91a] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 73d9508c-a07c-4c38-b5ab-5470ffc1d91a. [ 915.829164] nova-conductor[54394]: DEBUG nova.network.neutron [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] [instance: 73d9508c-a07c-4c38-b5ab-5470ffc1d91a] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 915.996883] nova-conductor[54394]: DEBUG nova.network.neutron [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] [instance: 73d9508c-a07c-4c38-b5ab-5470ffc1d91a] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 916.001488] nova-conductor[54394]: DEBUG nova.network.neutron [None req-d9f11dcf-ff90-4239-9994-30ef765080c0 tempest-ServersNegativeTestJSON-1589143891 tempest-ServersNegativeTestJSON-1589143891-project-member] [instance: 73d9508c-a07c-4c38-b5ab-5470ffc1d91a] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 916.235933] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell0:SELECT=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 916.257914] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=3,nova_cell1:INSERT=7 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 916.266158] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=6,nova_api:UPDATE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 916.274015] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=6,nova_api:DELETE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 917.467926] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: ca3f90c9-0ea2-4c63-8009-b875d65537e5] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 34ee2b83-6f90-46ae-ab06-517bb07c4933, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance ca3f90c9-0ea2-4c63-8009-b875d65537e5 was re-scheduled: Binding failed for port 34ee2b83-6f90-46ae-ab06-517bb07c4933, please check neutron logs for more information.\n'] [ 917.468585] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 917.468813] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ca3f90c9-0ea2-4c63-8009-b875d65537e5.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ca3f90c9-0ea2-4c63-8009-b875d65537e5. [ 917.469013] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: ca3f90c9-0ea2-4c63-8009-b875d65537e5] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ca3f90c9-0ea2-4c63-8009-b875d65537e5. [ 917.521769] nova-conductor[54394]: DEBUG nova.network.neutron [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: ca3f90c9-0ea2-4c63-8009-b875d65537e5] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 917.559046] nova-conductor[54394]: DEBUG nova.network.neutron [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: ca3f90c9-0ea2-4c63-8009-b875d65537e5] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 917.567521] nova-conductor[54394]: DEBUG nova.network.neutron [None req-aead9f8e-7be0-4ab0-8af6-1b8ad301e138 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: ca3f90c9-0ea2-4c63-8009-b875d65537e5] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 920.041105] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Expecting reply to msg 525b6e5e87f64351b1e48dd9c9e0fa82 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 920.188656] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 525b6e5e87f64351b1e48dd9c9e0fa82 [ 920.548376] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 920.567970] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 920.570815] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.003s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 920.570994] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 920.606090] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 920.606309] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 920.606470] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 920.606861] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 920.607037] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 920.607192] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 920.616742] nova-conductor[54394]: DEBUG nova.quota [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Getting quotas for project d10ee703bf3c4affa5b72621c111f180. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 920.619813] nova-conductor[54394]: DEBUG nova.quota [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Getting quotas for user 27e9284d43df4bd097c7b62f603c84ad and project d10ee703bf3c4affa5b72621c111f180. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 920.625224] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: b6e7fa9b-bb51-46ad-90b3-5130fc703872] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 920.625654] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 920.625844] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 920.625998] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 920.630033] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: b6e7fa9b-bb51-46ad-90b3-5130fc703872] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 920.630645] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 920.630831] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 920.631021] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 920.647482] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 920.647920] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 920.648136] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 921.585658] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Expecting reply to msg 69d9b7db39e24313ad964151f8934282 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 921.706070] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 69d9b7db39e24313ad964151f8934282 [ 922.088443] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 922.104863] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 922.106476] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.003s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 922.106766] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 922.139940] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 922.140835] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 922.141102] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 922.141568] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 922.141828] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 922.142052] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 922.160237] nova-conductor[54393]: DEBUG nova.quota [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Getting quotas for project bcfb0d202ded4b6faa8b7ee8812ab777. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 922.163125] nova-conductor[54393]: DEBUG nova.quota [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Getting quotas for user a9d53e1ff2db4ab7a617221507a9ab7b and project bcfb0d202ded4b6faa8b7ee8812ab777. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 922.168576] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 2b288d31-899a-44d9-ab2b-3df483b38452] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 922.169062] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 922.169268] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 922.169437] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 922.174368] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 2b288d31-899a-44d9-ab2b-3df483b38452] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 922.175240] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 922.175468] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 922.175623] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 922.191194] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 922.191404] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 922.191570] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 923.632592] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: ece742e3-4af5-4e24-ba29-c37f77c70e25] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 8ddb92ec-0e63-4290-8f07-48bdf2caa67a, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance ece742e3-4af5-4e24-ba29-c37f77c70e25 was re-scheduled: Binding failed for port 8ddb92ec-0e63-4290-8f07-48bdf2caa67a, please check neutron logs for more information.\n'] [ 923.633296] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 923.633430] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ece742e3-4af5-4e24-ba29-c37f77c70e25.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ece742e3-4af5-4e24-ba29-c37f77c70e25. [ 923.633912] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: ece742e3-4af5-4e24-ba29-c37f77c70e25] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ece742e3-4af5-4e24-ba29-c37f77c70e25. [ 923.660154] nova-conductor[54394]: DEBUG nova.network.neutron [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: ece742e3-4af5-4e24-ba29-c37f77c70e25] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 923.683028] nova-conductor[54394]: DEBUG nova.network.neutron [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: ece742e3-4af5-4e24-ba29-c37f77c70e25] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 923.686558] nova-conductor[54394]: DEBUG nova.network.neutron [None req-e37cef2f-5fdf-4ea8-929f-f36e6546c833 tempest-AttachInterfacesTestJSON-591934712 tempest-AttachInterfacesTestJSON-591934712-project-member] [instance: ece742e3-4af5-4e24-ba29-c37f77c70e25] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 928.996765] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Expecting reply to msg bc447e86266740518115f952711b0a8a in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 929.143557] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg bc447e86266740518115f952711b0a8a [ 929.498777] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Took 0.50 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 929.516552] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 929.516824] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 929.517005] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 929.592689] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 929.592830] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 929.592999] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 929.593429] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 929.593568] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 929.593691] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 929.608440] nova-conductor[54394]: DEBUG nova.quota [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Getting quotas for project 1d837b5b0126427eb945c5d383005912. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 929.628523] nova-conductor[54394]: DEBUG nova.quota [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Getting quotas for user 521e4565371f4da0bd61b666f21cbe3d and project 1d837b5b0126427eb945c5d383005912. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 929.637201] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] [instance: b4baa654-30a9-495f-89a4-120936ab2391] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 929.638086] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 929.638838] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 929.639271] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 929.644170] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] [instance: b4baa654-30a9-495f-89a4-120936ab2391] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 929.645136] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 929.645489] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 929.645981] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 929.662224] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 929.662608] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 929.662944] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 930.649985] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] [instance: c34de847-9df4-4393-b691-a689a24b9a3f] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port f195aef8-54f4-493c-a2ae-1d10a674ec41, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance c34de847-9df4-4393-b691-a689a24b9a3f was re-scheduled: Binding failed for port f195aef8-54f4-493c-a2ae-1d10a674ec41, please check neutron logs for more information.\n'] [ 930.651197] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 930.651376] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance c34de847-9df4-4393-b691-a689a24b9a3f.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance c34de847-9df4-4393-b691-a689a24b9a3f. [ 930.651718] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] [instance: c34de847-9df4-4393-b691-a689a24b9a3f] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance c34de847-9df4-4393-b691-a689a24b9a3f. [ 930.681266] nova-conductor[54393]: DEBUG nova.network.neutron [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] [instance: c34de847-9df4-4393-b691-a689a24b9a3f] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 930.711583] nova-conductor[54393]: DEBUG nova.network.neutron [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] [instance: c34de847-9df4-4393-b691-a689a24b9a3f] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 930.718608] nova-conductor[54393]: DEBUG nova.network.neutron [None req-e3e88b45-28b9-4fc6-a264-d7ae9b96b8e7 tempest-InstanceActionsNegativeTestJSON-987561372 tempest-InstanceActionsNegativeTestJSON-987561372-project-member] [instance: c34de847-9df4-4393-b691-a689a24b9a3f] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 932.146656] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell0:SELECT=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 932.178276] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=3,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=8,nova_cell1:RELEASE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 932.186908] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=8,nova_api:UPDATE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 932.187722] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=4,nova_api:DELETE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 935.873027] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=358,nova_cell1:UPDATE=190,nova_cell1:INSERT=22,nova_cell1:SAVEPOINT=42,nova_cell1:RELEASE=42,nova_cell1:DELETE=4 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 936.226143] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: 98f26762-8ec1-4925-9a0a-5e25df3e7934] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port d6809fa7-509e-4a48-ba11-82858d7c050f, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 98f26762-8ec1-4925-9a0a-5e25df3e7934 was re-scheduled: Binding failed for port d6809fa7-509e-4a48-ba11-82858d7c050f, please check neutron logs for more information.\n'] [ 936.226690] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 936.226973] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 98f26762-8ec1-4925-9a0a-5e25df3e7934.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 98f26762-8ec1-4925-9a0a-5e25df3e7934. [ 936.227287] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: 98f26762-8ec1-4925-9a0a-5e25df3e7934] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 98f26762-8ec1-4925-9a0a-5e25df3e7934. [ 936.252884] nova-conductor[54394]: DEBUG nova.network.neutron [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: 98f26762-8ec1-4925-9a0a-5e25df3e7934] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 936.267984] nova-conductor[54394]: DEBUG nova.network.neutron [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: 98f26762-8ec1-4925-9a0a-5e25df3e7934] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 936.270617] nova-conductor[54394]: DEBUG nova.network.neutron [None req-216a4dee-cc25-4914-af03-0c5e12d4e840 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: 98f26762-8ec1-4925-9a0a-5e25df3e7934] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 937.368241] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Expecting reply to msg fad41c0c62fe4d26be2718de631b894c in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 937.481936] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fad41c0c62fe4d26be2718de631b894c [ 937.870902] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 937.883939] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 937.883939] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 937.884144] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 937.915229] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: c142f8c1-93e7-4a3e-9c44-9998b9a800a2] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 44e25eae-4c6a-48f0-a3ce-504fb950889e, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance c142f8c1-93e7-4a3e-9c44-9998b9a800a2 was re-scheduled: Binding failed for port 44e25eae-4c6a-48f0-a3ce-504fb950889e, please check neutron logs for more information.\n'] [ 937.916200] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 937.916504] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance c142f8c1-93e7-4a3e-9c44-9998b9a800a2.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance c142f8c1-93e7-4a3e-9c44-9998b9a800a2. [ 937.916769] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: c142f8c1-93e7-4a3e-9c44-9998b9a800a2] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance c142f8c1-93e7-4a3e-9c44-9998b9a800a2. [ 937.940534] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 937.940801] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 937.941150] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 937.941561] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 937.941771] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 937.941952] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 937.947388] nova-conductor[54394]: DEBUG nova.network.neutron [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: c142f8c1-93e7-4a3e-9c44-9998b9a800a2] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 937.954873] nova-conductor[54394]: DEBUG nova.quota [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Getting quotas for project 7b6177ee33c74f3a89ab5251a94038e7. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 937.958104] nova-conductor[54394]: DEBUG nova.quota [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Getting quotas for user d1429ed3916c42af92467fd7255cb2ec and project 7b6177ee33c74f3a89ab5251a94038e7. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 937.963545] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: f4f80b12-e74a-40c1-b66d-a099b91344c7] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 937.964132] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 937.964447] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 937.964668] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 937.966397] nova-conductor[54394]: DEBUG nova.network.neutron [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: c142f8c1-93e7-4a3e-9c44-9998b9a800a2] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 937.969502] nova-conductor[54394]: DEBUG nova.network.neutron [None req-92e4d18d-ecc7-4e68-ba56-132e2e9dd29a tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: c142f8c1-93e7-4a3e-9c44-9998b9a800a2] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 937.972089] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: f4f80b12-e74a-40c1-b66d-a099b91344c7] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 937.972794] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 937.973028] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 937.973204] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 937.996490] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 937.996756] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 937.996920] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 938.708068] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Expecting reply to msg f7dc2462f40c49ae89e03a5cbbe4ed83 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 938.828987] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f7dc2462f40c49ae89e03a5cbbe4ed83 [ 939.210517] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Took 0.50 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 939.222911] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 939.223144] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 939.223314] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 939.250640] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 939.250852] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 939.251022] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 939.251389] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 939.251573] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 939.251732] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 939.265878] nova-conductor[54393]: DEBUG nova.quota [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Getting quotas for project ac8618fb49594cebbbbb5dbd6969a4a3. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 939.265878] nova-conductor[54393]: DEBUG nova.quota [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Getting quotas for user b9aba9a68af34d3fb1466c85dfcb6940 and project ac8618fb49594cebbbbb5dbd6969a4a3. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 939.265878] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 85081615-3ca3-49aa-a9a2-e89727c47dc9] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 939.266222] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 939.266415] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 939.266578] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 939.271564] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 85081615-3ca3-49aa-a9a2-e89727c47dc9] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 939.272169] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 939.272366] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 939.272536] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 939.285355] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 939.285552] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 939.285717] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 939.587536] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: b6e7fa9b-bb51-46ad-90b3-5130fc703872] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port a50a0c1c-ab26-4968-9ca1-c9018cb91219, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance b6e7fa9b-bb51-46ad-90b3-5130fc703872 was re-scheduled: Binding failed for port a50a0c1c-ab26-4968-9ca1-c9018cb91219, please check neutron logs for more information.\n'] [ 939.588179] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 939.588420] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b6e7fa9b-bb51-46ad-90b3-5130fc703872.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b6e7fa9b-bb51-46ad-90b3-5130fc703872. [ 939.588856] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: b6e7fa9b-bb51-46ad-90b3-5130fc703872] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b6e7fa9b-bb51-46ad-90b3-5130fc703872. [ 939.609453] nova-conductor[54393]: DEBUG nova.network.neutron [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: b6e7fa9b-bb51-46ad-90b3-5130fc703872] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 939.637550] nova-conductor[54393]: DEBUG nova.network.neutron [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: b6e7fa9b-bb51-46ad-90b3-5130fc703872] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 939.641144] nova-conductor[54393]: DEBUG nova.network.neutron [None req-e83c791b-6a6b-49dd-8217-b569b0b7bc42 tempest-AttachVolumeNegativeTest-2098360093 tempest-AttachVolumeNegativeTest-2098360093-project-member] [instance: b6e7fa9b-bb51-46ad-90b3-5130fc703872] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 939.981982] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SAVEPOINT=35,nova_cell1:RELEASE=35,nova_cell1:SELECT=390,nova_cell1:UPDATE=199,nova_cell1:INSERT=26 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 943.365652] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 2b288d31-899a-44d9-ab2b-3df483b38452] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 8f86c889-c8f5-4fe1-966f-d48a0d27c5fd, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 2b288d31-899a-44d9-ab2b-3df483b38452 was re-scheduled: Binding failed for port 8f86c889-c8f5-4fe1-966f-d48a0d27c5fd, please check neutron logs for more information.\n'] [ 943.366271] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 943.366518] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 2b288d31-899a-44d9-ab2b-3df483b38452.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 2b288d31-899a-44d9-ab2b-3df483b38452. [ 943.366868] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 2b288d31-899a-44d9-ab2b-3df483b38452] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 2b288d31-899a-44d9-ab2b-3df483b38452. [ 943.401405] nova-conductor[54394]: DEBUG nova.network.neutron [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 2b288d31-899a-44d9-ab2b-3df483b38452] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 943.427427] nova-conductor[54394]: DEBUG nova.network.neutron [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 2b288d31-899a-44d9-ab2b-3df483b38452] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 943.430739] nova-conductor[54394]: DEBUG nova.network.neutron [None req-124b0ead-71a1-4013-a4df-ef62864b0b83 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 2b288d31-899a-44d9-ab2b-3df483b38452] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 944.636406] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Expecting reply to msg 539ee0c0b05e4f9d9ba6049bd0e2bdd4 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 944.759845] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 539ee0c0b05e4f9d9ba6049bd0e2bdd4 [ 945.138892] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 945.151104] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 945.151380] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 945.151517] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 945.181486] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 945.181714] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 945.181886] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 945.182250] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 945.182434] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 945.182624] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 945.189910] nova-conductor[54393]: DEBUG nova.quota [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Getting quotas for project bcfb0d202ded4b6faa8b7ee8812ab777. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 945.191958] nova-conductor[54393]: DEBUG nova.quota [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Getting quotas for user a9d53e1ff2db4ab7a617221507a9ab7b and project bcfb0d202ded4b6faa8b7ee8812ab777. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 945.197027] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: c1526963-ef0c-42f8-a6c1-ba77b436a5b4] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 945.197452] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 945.197647] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 945.197835] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 945.201696] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: c1526963-ef0c-42f8-a6c1-ba77b436a5b4] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 945.202313] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 945.202528] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 945.202694] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 945.214499] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 945.214737] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 945.214909] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 947.950074] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell0:SELECT=4 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 947.976304] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=16,nova_cell1:UPDATE=3,nova_cell1:INSERT=30,nova_cell1:SAVEPOINT=1,nova_cell1:RELEASE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 947.989711] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=21,nova_api:UPDATE=3,nova_api:DELETE=3 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 947.994623] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=29,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 949.266936] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] [instance: b4baa654-30a9-495f-89a4-120936ab2391] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port ef24df9c-0607-4c8c-aaa4-b4d43efc6c1b, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance b4baa654-30a9-495f-89a4-120936ab2391 was re-scheduled: Binding failed for port ef24df9c-0607-4c8c-aaa4-b4d43efc6c1b, please check neutron logs for more information.\n'] [ 949.267491] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 949.267730] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b4baa654-30a9-495f-89a4-120936ab2391.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b4baa654-30a9-495f-89a4-120936ab2391. [ 949.268016] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] [instance: b4baa654-30a9-495f-89a4-120936ab2391] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b4baa654-30a9-495f-89a4-120936ab2391. [ 949.295917] nova-conductor[54394]: DEBUG nova.network.neutron [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] [instance: b4baa654-30a9-495f-89a4-120936ab2391] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 949.321557] nova-conductor[54394]: DEBUG nova.network.neutron [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] [instance: b4baa654-30a9-495f-89a4-120936ab2391] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 949.324610] nova-conductor[54394]: DEBUG nova.network.neutron [None req-33024b6f-f501-4194-b451-8681c9ebedd9 tempest-ServerTagsTestJSON-345145739 tempest-ServerTagsTestJSON-345145739-project-member] [instance: b4baa654-30a9-495f-89a4-120936ab2391] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 953.261548] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: f4f80b12-e74a-40c1-b66d-a099b91344c7] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port c4a74bb4-4ae7-4240-8dfb-631162b104db, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance f4f80b12-e74a-40c1-b66d-a099b91344c7 was re-scheduled: Binding failed for port c4a74bb4-4ae7-4240-8dfb-631162b104db, please check neutron logs for more information.\n'] [ 953.262607] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 953.262995] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance f4f80b12-e74a-40c1-b66d-a099b91344c7.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance f4f80b12-e74a-40c1-b66d-a099b91344c7. [ 953.263469] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: f4f80b12-e74a-40c1-b66d-a099b91344c7] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance f4f80b12-e74a-40c1-b66d-a099b91344c7. [ 953.285639] nova-conductor[54393]: DEBUG nova.network.neutron [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: f4f80b12-e74a-40c1-b66d-a099b91344c7] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 953.301065] nova-conductor[54393]: DEBUG nova.network.neutron [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: f4f80b12-e74a-40c1-b66d-a099b91344c7] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 953.303746] nova-conductor[54393]: DEBUG nova.network.neutron [None req-16998dd8-3bd9-486d-a4c7-aff90b014ab7 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: f4f80b12-e74a-40c1-b66d-a099b91344c7] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 953.928864] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Expecting reply to msg 7bc69939c5f54a0db64946dbe41263c9 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 954.044956] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7bc69939c5f54a0db64946dbe41263c9 [ 954.431042] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Took 0.50 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 954.443777] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 954.444017] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 954.444213] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 954.469589] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 954.469803] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 954.469976] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 954.470332] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 954.470514] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 954.470671] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 954.479738] nova-conductor[54394]: DEBUG nova.quota [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Getting quotas for project 7b6177ee33c74f3a89ab5251a94038e7. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 954.482508] nova-conductor[54394]: DEBUG nova.quota [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Getting quotas for user d1429ed3916c42af92467fd7255cb2ec and project 7b6177ee33c74f3a89ab5251a94038e7. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 954.487254] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: f3195ca5-b9e1-4acd-8779-f9939ae5f07e] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 954.487645] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 954.487845] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 954.488021] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 954.491761] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: f3195ca5-b9e1-4acd-8779-f9939ae5f07e] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 954.492382] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 954.492580] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 954.492744] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 954.506022] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 954.506180] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 954.506347] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 954.820986] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 85081615-3ca3-49aa-a9a2-e89727c47dc9] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port ca8f787e-8eda-4c51-ad23-e3c7113e1582, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 85081615-3ca3-49aa-a9a2-e89727c47dc9 was re-scheduled: Binding failed for port ca8f787e-8eda-4c51-ad23-e3c7113e1582, please check neutron logs for more information.\n'] [ 954.821536] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 954.821751] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 85081615-3ca3-49aa-a9a2-e89727c47dc9.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 85081615-3ca3-49aa-a9a2-e89727c47dc9. [ 954.821952] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 85081615-3ca3-49aa-a9a2-e89727c47dc9] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 85081615-3ca3-49aa-a9a2-e89727c47dc9. [ 954.841913] nova-conductor[54393]: DEBUG nova.network.neutron [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 85081615-3ca3-49aa-a9a2-e89727c47dc9] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 954.855973] nova-conductor[54393]: DEBUG nova.network.neutron [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 85081615-3ca3-49aa-a9a2-e89727c47dc9] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 954.858988] nova-conductor[54393]: DEBUG nova.network.neutron [None req-702b6bdc-1550-4b9e-b368-d7744857dbe8 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 85081615-3ca3-49aa-a9a2-e89727c47dc9] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 955.187270] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell0:SELECT=2 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 955.205373] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=6,nova_cell1:INSERT=14 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 955.211440] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 955.213430] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 955.569037] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Expecting reply to msg 33ddd78cf4b348109d5b54f69919f57b in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 955.680340] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 33ddd78cf4b348109d5b54f69919f57b [ 956.071836] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 956.083606] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 956.083825] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 956.083998] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 956.109681] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 956.109903] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 956.110074] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 956.110454] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 956.110634] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 956.110793] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 956.117815] nova-conductor[54393]: DEBUG nova.quota [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Getting quotas for project ac8618fb49594cebbbbb5dbd6969a4a3. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 956.119845] nova-conductor[54393]: DEBUG nova.quota [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Getting quotas for user b9aba9a68af34d3fb1466c85dfcb6940 and project ac8618fb49594cebbbbb5dbd6969a4a3. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 956.124555] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 7901f729-a43e-413e-becc-a8796f66214c] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 956.124982] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 956.125175] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 956.125337] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 956.129398] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 7901f729-a43e-413e-becc-a8796f66214c] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 956.129992] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 956.130190] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 956.130347] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 956.141852] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 956.142083] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 956.142221] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 960.112064] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: c1526963-ef0c-42f8-a6c1-ba77b436a5b4] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port c14932bc-5af9-4bef-8446-c3f0dae61211, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance c1526963-ef0c-42f8-a6c1-ba77b436a5b4 was re-scheduled: Binding failed for port c14932bc-5af9-4bef-8446-c3f0dae61211, please check neutron logs for more information.\n'] [ 960.112680] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 960.112915] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance c1526963-ef0c-42f8-a6c1-ba77b436a5b4.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance c1526963-ef0c-42f8-a6c1-ba77b436a5b4. [ 960.113220] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: c1526963-ef0c-42f8-a6c1-ba77b436a5b4] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance c1526963-ef0c-42f8-a6c1-ba77b436a5b4. [ 960.133981] nova-conductor[54393]: DEBUG nova.network.neutron [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: c1526963-ef0c-42f8-a6c1-ba77b436a5b4] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 960.149973] nova-conductor[54393]: DEBUG nova.network.neutron [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: c1526963-ef0c-42f8-a6c1-ba77b436a5b4] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 960.153287] nova-conductor[54393]: DEBUG nova.network.neutron [None req-b1cc98fa-b79f-4353-a460-9d347476cc94 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: c1526963-ef0c-42f8-a6c1-ba77b436a5b4] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 961.214527] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Expecting reply to msg f843cb078ee846c8bb259e1691d371f3 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 961.331119] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f843cb078ee846c8bb259e1691d371f3 [ 961.717159] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 961.729725] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 961.729963] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 961.730195] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 961.765871] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 961.766105] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 961.766272] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 961.766641] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 961.766826] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 961.766981] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 961.775918] nova-conductor[54393]: DEBUG nova.quota [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Getting quotas for project bcfb0d202ded4b6faa8b7ee8812ab777. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 961.779307] nova-conductor[54393]: DEBUG nova.quota [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Getting quotas for user a9d53e1ff2db4ab7a617221507a9ab7b and project bcfb0d202ded4b6faa8b7ee8812ab777. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 961.784025] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 085f0124-9406-4fff-b981-3f1e5a4d8835] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 961.784448] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 961.784646] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 961.784810] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 961.790771] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 085f0124-9406-4fff-b981-3f1e5a4d8835] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 961.791407] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 961.791602] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 961.791766] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 961.807530] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 961.807776] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 961.807963] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 964.474958] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell0:SELECT=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 964.494589] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=3,nova_cell1:INSERT=7 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 964.501855] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=7 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 964.503832] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=5,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 969.802348] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: f3195ca5-b9e1-4acd-8779-f9939ae5f07e] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 849279d5-a956-497d-b9db-94ad1ba80337, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance f3195ca5-b9e1-4acd-8779-f9939ae5f07e was re-scheduled: Binding failed for port 849279d5-a956-497d-b9db-94ad1ba80337, please check neutron logs for more information.\n'] [ 969.802916] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 969.803142] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance f3195ca5-b9e1-4acd-8779-f9939ae5f07e.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance f3195ca5-b9e1-4acd-8779-f9939ae5f07e. [ 969.803465] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: f3195ca5-b9e1-4acd-8779-f9939ae5f07e] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance f3195ca5-b9e1-4acd-8779-f9939ae5f07e. [ 969.825891] nova-conductor[54394]: DEBUG nova.network.neutron [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: f3195ca5-b9e1-4acd-8779-f9939ae5f07e] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 969.841411] nova-conductor[54394]: DEBUG nova.network.neutron [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: f3195ca5-b9e1-4acd-8779-f9939ae5f07e] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 969.845040] nova-conductor[54394]: DEBUG nova.network.neutron [None req-4b168c85-78ad-4db2-b83e-e4509d5a7361 tempest-DeleteServersTestJSON-5573112 tempest-DeleteServersTestJSON-5573112-project-member] [instance: f3195ca5-b9e1-4acd-8779-f9939ae5f07e] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 971.430867] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 7901f729-a43e-413e-becc-a8796f66214c] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 50724de8-a5a6-48d8-883d-90f56fe40411, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 7901f729-a43e-413e-becc-a8796f66214c was re-scheduled: Binding failed for port 50724de8-a5a6-48d8-883d-90f56fe40411, please check neutron logs for more information.\n'] [ 971.431502] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 971.431743] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7901f729-a43e-413e-becc-a8796f66214c.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7901f729-a43e-413e-becc-a8796f66214c. [ 971.431946] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 7901f729-a43e-413e-becc-a8796f66214c] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7901f729-a43e-413e-becc-a8796f66214c. [ 971.469574] nova-conductor[54394]: DEBUG nova.network.neutron [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 7901f729-a43e-413e-becc-a8796f66214c] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 971.496367] nova-conductor[54394]: DEBUG nova.network.neutron [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 7901f729-a43e-413e-becc-a8796f66214c] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 971.500219] nova-conductor[54394]: DEBUG nova.network.neutron [None req-c65dcd1a-9985-4d85-a0f9-c7434ec05917 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 7901f729-a43e-413e-becc-a8796f66214c] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 971.770835] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell0:SELECT=2 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 971.793970] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=6,nova_cell1:INSERT=14 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 971.801810] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=2,nova_api:DELETE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 971.806162] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 973.170981] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Expecting reply to msg 97d42e932fdd4d08b1871d204973a282 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 973.298018] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 97d42e932fdd4d08b1871d204973a282 [ 973.673711] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 973.702382] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 973.702623] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 973.702781] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 973.746144] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 973.746358] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 973.746516] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 973.746899] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 973.747110] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 973.747274] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 973.755356] nova-conductor[54393]: DEBUG nova.quota [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Getting quotas for project ac8618fb49594cebbbbb5dbd6969a4a3. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 973.758493] nova-conductor[54393]: DEBUG nova.quota [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Getting quotas for user b9aba9a68af34d3fb1466c85dfcb6940 and project ac8618fb49594cebbbbb5dbd6969a4a3. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 973.763461] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 1afc934e-442c-4c03-81e5-8e7d925a0f6b] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 973.763942] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 973.764166] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 973.764333] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 973.768726] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 1afc934e-442c-4c03-81e5-8e7d925a0f6b] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 973.769349] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 973.769648] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 973.769703] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 973.795514] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 973.795752] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 973.795927] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 974.274695] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Expecting reply to msg 4e13b7e29be44203bc2b9a04414adfcb in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 974.406575] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4e13b7e29be44203bc2b9a04414adfcb [ 974.640889] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Expecting reply to msg 3510452c70584562a17c54d68cf2e859 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 974.715754] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 085f0124-9406-4fff-b981-3f1e5a4d8835] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port b6994dee-e7b7-4c6c-b450-c46abcdc16bd, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 085f0124-9406-4fff-b981-3f1e5a4d8835 was re-scheduled: Binding failed for port b6994dee-e7b7-4c6c-b450-c46abcdc16bd, please check neutron logs for more information.\n'] [ 974.715754] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 974.715754] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 085f0124-9406-4fff-b981-3f1e5a4d8835.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 085f0124-9406-4fff-b981-3f1e5a4d8835. [ 974.715754] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 085f0124-9406-4fff-b981-3f1e5a4d8835] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 085f0124-9406-4fff-b981-3f1e5a4d8835. [ 974.755136] nova-conductor[54393]: DEBUG nova.network.neutron [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 085f0124-9406-4fff-b981-3f1e5a4d8835] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 974.770221] nova-conductor[54393]: DEBUG nova.network.neutron [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 085f0124-9406-4fff-b981-3f1e5a4d8835] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 974.773688] nova-conductor[54393]: DEBUG nova.network.neutron [None req-8f9e1620-4b3c-48e2-b232-11e7569b917b tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 085f0124-9406-4fff-b981-3f1e5a4d8835] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 974.777965] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 974.791001] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 974.791231] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 974.791397] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 974.793710] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3510452c70584562a17c54d68cf2e859 [ 974.817215] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 974.817438] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 974.817603] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 974.817986] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 974.818166] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 974.818321] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 974.825395] nova-conductor[54394]: DEBUG nova.quota [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Getting quotas for project a3be485085b548348ba5f5a36c24628f. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 974.827892] nova-conductor[54394]: DEBUG nova.quota [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Getting quotas for user 32bcdfde286149dc888f7605fe53b81d and project a3be485085b548348ba5f5a36c24628f. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 974.832590] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] [instance: 1976dfb9-0511-494b-9865-d36fd8b9eeed] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 974.832995] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 974.833187] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 974.833349] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 974.835907] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] [instance: 1976dfb9-0511-494b-9865-d36fd8b9eeed] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 974.836541] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 974.836773] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 974.836922] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 974.876173] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 974.876434] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 974.876606] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 975.145444] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Took 0.50 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 975.155233] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 975.155955] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 975.156917] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 975.190585] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 975.190823] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 975.190990] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 975.191457] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 975.191682] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 975.191847] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 975.199705] nova-conductor[54394]: DEBUG nova.quota [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Getting quotas for project a3be485085b548348ba5f5a36c24628f. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 975.202002] nova-conductor[54394]: DEBUG nova.quota [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Getting quotas for user 32bcdfde286149dc888f7605fe53b81d and project a3be485085b548348ba5f5a36c24628f. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 975.207506] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] [instance: a2f79d0b-27f5-4969-8b9e-d5a500cd4ca0] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 975.208082] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 975.208341] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 975.208438] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 975.221478] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] [instance: a2f79d0b-27f5-4969-8b9e-d5a500cd4ca0] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 975.222428] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 975.223246] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 975.223513] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 975.248088] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 975.248335] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 975.248505] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 975.631073] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Expecting reply to msg ccbbc95b98244b29859d9a061e7de808 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 975.794169] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ccbbc95b98244b29859d9a061e7de808 [ 976.134076] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 976.146326] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 976.146509] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 976.146679] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 976.167563] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 976.167784] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 976.167927] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 976.168309] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 976.168490] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 976.168645] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 976.175392] nova-conductor[54394]: DEBUG nova.quota [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Getting quotas for project bcfb0d202ded4b6faa8b7ee8812ab777. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 976.177430] nova-conductor[54394]: DEBUG nova.quota [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Getting quotas for user a9d53e1ff2db4ab7a617221507a9ab7b and project bcfb0d202ded4b6faa8b7ee8812ab777. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 976.182106] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 8e6b3d4a-5a98-444c-95f9-df40db866b07] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 976.182520] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 976.182709] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 976.182871] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 976.191534] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 8e6b3d4a-5a98-444c-95f9-df40db866b07] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 976.192147] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 976.192342] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 976.192506] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 976.207833] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 976.208043] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 976.208216] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 983.751414] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell0:SELECT=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 983.771223] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=3,nova_cell1:INSERT=7 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 983.786929] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=5,nova_api:UPDATE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 983.791900] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=7,nova_api:DELETE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 986.172258] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell0:SELECT=3 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 986.194190] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=9,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=22,nova_cell1:RELEASE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 986.203203] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=18,nova_api:UPDATE=2,nova_api:DELETE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 986.207219] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=18,nova_api:DELETE=2,nova_api:UPDATE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 991.204032] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 1afc934e-442c-4c03-81e5-8e7d925a0f6b] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port e4cddc48-3774-4011-bf91-ac963f205dd7, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 1afc934e-442c-4c03-81e5-8e7d925a0f6b was re-scheduled: Binding failed for port e4cddc48-3774-4011-bf91-ac963f205dd7, please check neutron logs for more information.\n'] [ 991.204489] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 991.204736] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1afc934e-442c-4c03-81e5-8e7d925a0f6b.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1afc934e-442c-4c03-81e5-8e7d925a0f6b. [ 991.204960] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 1afc934e-442c-4c03-81e5-8e7d925a0f6b] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1afc934e-442c-4c03-81e5-8e7d925a0f6b. [ 991.225853] nova-conductor[54393]: DEBUG nova.network.neutron [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 1afc934e-442c-4c03-81e5-8e7d925a0f6b] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 991.246613] nova-conductor[54393]: DEBUG nova.network.neutron [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 1afc934e-442c-4c03-81e5-8e7d925a0f6b] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 991.249555] nova-conductor[54393]: DEBUG nova.network.neutron [None req-9f8d93fa-4d61-407c-8183-b7aa47536cd1 tempest-ServersTestJSON-951823456 tempest-ServersTestJSON-951823456-project-member] [instance: 1afc934e-442c-4c03-81e5-8e7d925a0f6b] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 992.831408] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] [instance: 1976dfb9-0511-494b-9865-d36fd8b9eeed] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 34149bc7-3907-4f72-be29-88539466d653, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 1976dfb9-0511-494b-9865-d36fd8b9eeed was re-scheduled: Binding failed for port 34149bc7-3907-4f72-be29-88539466d653, please check neutron logs for more information.\n'] [ 992.831767] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 992.831996] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1976dfb9-0511-494b-9865-d36fd8b9eeed.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1976dfb9-0511-494b-9865-d36fd8b9eeed. [ 992.832848] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] [instance: 1976dfb9-0511-494b-9865-d36fd8b9eeed] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1976dfb9-0511-494b-9865-d36fd8b9eeed. [ 992.861357] nova-conductor[54394]: DEBUG nova.network.neutron [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] [instance: 1976dfb9-0511-494b-9865-d36fd8b9eeed] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 992.876229] nova-conductor[54394]: DEBUG nova.network.neutron [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] [instance: 1976dfb9-0511-494b-9865-d36fd8b9eeed] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 992.879396] nova-conductor[54394]: DEBUG nova.network.neutron [None req-259cacf9-d9d3-48d7-9eaf-c2bdfbb45441 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] [instance: 1976dfb9-0511-494b-9865-d36fd8b9eeed] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 994.495494] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] [instance: a2f79d0b-27f5-4969-8b9e-d5a500cd4ca0] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 0f64bf0f-b8a7-45ef-ba80-355838589eb8, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance a2f79d0b-27f5-4969-8b9e-d5a500cd4ca0 was re-scheduled: Binding failed for port 0f64bf0f-b8a7-45ef-ba80-355838589eb8, please check neutron logs for more information.\n'] [ 994.496148] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 994.496466] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance a2f79d0b-27f5-4969-8b9e-d5a500cd4ca0.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance a2f79d0b-27f5-4969-8b9e-d5a500cd4ca0. [ 994.496805] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] [instance: a2f79d0b-27f5-4969-8b9e-d5a500cd4ca0] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance a2f79d0b-27f5-4969-8b9e-d5a500cd4ca0. [ 994.521489] nova-conductor[54394]: DEBUG nova.network.neutron [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] [instance: a2f79d0b-27f5-4969-8b9e-d5a500cd4ca0] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 994.538673] nova-conductor[54394]: DEBUG nova.network.neutron [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] [instance: a2f79d0b-27f5-4969-8b9e-d5a500cd4ca0] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 994.541857] nova-conductor[54394]: DEBUG nova.network.neutron [None req-86adec84-a41a-48ca-8e6a-b66c82ba66c5 tempest-ServerRescueNegativeTestJSON-1985616379 tempest-ServerRescueNegativeTestJSON-1985616379-project-member] [instance: a2f79d0b-27f5-4969-8b9e-d5a500cd4ca0] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 996.120294] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 8e6b3d4a-5a98-444c-95f9-df40db866b07] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 0502eb97-28cc-4967-b395-bae99550ec61, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 8e6b3d4a-5a98-444c-95f9-df40db866b07 was re-scheduled: Binding failed for port 0502eb97-28cc-4967-b395-bae99550ec61, please check neutron logs for more information.\n'] [ 996.121000] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 996.121296] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8e6b3d4a-5a98-444c-95f9-df40db866b07.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8e6b3d4a-5a98-444c-95f9-df40db866b07. [ 996.121464] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=329,nova_cell1:UPDATE=151,nova_cell1:INSERT=24,nova_cell1:SAVEPOINT=35,nova_cell1:RELEASE=35 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 996.121649] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 8e6b3d4a-5a98-444c-95f9-df40db866b07] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8e6b3d4a-5a98-444c-95f9-df40db866b07. [ 996.143702] nova-conductor[54393]: DEBUG nova.network.neutron [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 8e6b3d4a-5a98-444c-95f9-df40db866b07] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 996.158794] nova-conductor[54393]: DEBUG nova.network.neutron [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 8e6b3d4a-5a98-444c-95f9-df40db866b07] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 996.163015] nova-conductor[54393]: DEBUG nova.network.neutron [None req-6c7e8786-ed8e-40f6-99e4-72c89bb08b7e tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 8e6b3d4a-5a98-444c-95f9-df40db866b07] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 997.585771] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Expecting reply to msg 3fd9e680aedf4c4f9e91f7c589267090 in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 997.703745] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Expecting reply to msg 582d7bc359a24357917fb169f6038b98 in queue reply_61d6f4d8c46c471b8f9eea8cf9836b6e [ 997.736369] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3fd9e680aedf4c4f9e91f7c589267090 [ 997.814516] nova-conductor[54394]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 582d7bc359a24357917fb169f6038b98 [ 998.088899] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 998.101077] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 998.101287] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 998.101449] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 998.140732] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 998.141012] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 998.141175] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 998.141537] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 998.142031] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 998.142230] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 998.150907] nova-conductor[54393]: DEBUG nova.quota [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Getting quotas for project 3332aa3d846f417e9a53239cadac267f. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 998.153038] nova-conductor[54393]: DEBUG nova.quota [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Getting quotas for user 8a5c84e9df8543279986786a0174f8d5 and project 3332aa3d846f417e9a53239cadac267f. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 998.157910] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] [instance: 16ec58d0-8b40-490c-868f-d11052c537c3] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 998.158335] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 998.158533] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 998.158698] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 998.165487] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] [instance: 16ec58d0-8b40-490c-868f-d11052c537c3] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 998.166271] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 998.166475] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 998.166637] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 998.186711] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 998.186914] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 998.187112] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 998.209154] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54394) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 998.223922] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 998.224513] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 998.225104] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 998.264278] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 998.264840] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 998.265147] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 998.265639] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 998.265961] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 998.266271] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 998.274783] nova-conductor[54394]: DEBUG nova.quota [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Getting quotas for project bcfb0d202ded4b6faa8b7ee8812ab777. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 998.277887] nova-conductor[54394]: DEBUG nova.quota [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Getting quotas for user a9d53e1ff2db4ab7a617221507a9ab7b and project bcfb0d202ded4b6faa8b7ee8812ab777. Resources: {'ram', 'instances', 'cores'} {{(pid=54394) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 998.283340] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 415fb59a-a9db-424d-b066-03ead9103131] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54394) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 998.283889] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 998.284224] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 998.284531] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 998.301472] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 415fb59a-a9db-424d-b066-03ead9103131] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54394) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 998.302372] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 998.302720] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 998.303011] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 998.316330] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 998.316544] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 998.316705] nova-conductor[54394]: DEBUG oslo_concurrency.lockutils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54394) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1000.337987] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:UPDATE=164,nova_cell1:SELECT=331,nova_cell1:INSERT=24,nova_cell1:SAVEPOINT=38,nova_cell1:RELEASE=38 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1008.147065] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell0:SELECT=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1008.168165] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=3,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=8,nova_cell1:RELEASE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1008.177826] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=6,nova_api:UPDATE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1008.182213] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=6,nova_api:DELETE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1008.269964] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell0:SELECT=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1008.306164] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=3,nova_cell1:INSERT=7 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1008.313152] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=7 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1008.315081] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_api:SELECT=5,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1010.821927] nova-conductor[54394]: ERROR nova.scheduler.utils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] [instance: 16ec58d0-8b40-490c-868f-d11052c537c3] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 8a1345f6-8705-4b0d-81f2-e19c8116f7f7, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 16ec58d0-8b40-490c-868f-d11052c537c3 was re-scheduled: Binding failed for port 8a1345f6-8705-4b0d-81f2-e19c8116f7f7, please check neutron logs for more information.\n'] [ 1010.822485] nova-conductor[54394]: DEBUG nova.conductor.manager [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Rescheduling: True {{(pid=54394) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 1010.822708] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 16ec58d0-8b40-490c-868f-d11052c537c3.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 16ec58d0-8b40-490c-868f-d11052c537c3. [ 1010.823037] nova-conductor[54394]: WARNING nova.scheduler.utils [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] [instance: 16ec58d0-8b40-490c-868f-d11052c537c3] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 16ec58d0-8b40-490c-868f-d11052c537c3. [ 1010.843715] nova-conductor[54394]: DEBUG nova.network.neutron [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] [instance: 16ec58d0-8b40-490c-868f-d11052c537c3] deallocate_for_instance() {{(pid=54394) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1010.859185] nova-conductor[54394]: DEBUG nova.network.neutron [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] [instance: 16ec58d0-8b40-490c-868f-d11052c537c3] Instance cache missing network info. {{(pid=54394) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1010.861842] nova-conductor[54394]: DEBUG nova.network.neutron [None req-7b0ccb87-3b45-49e6-be02-035956f771ce tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] [instance: 16ec58d0-8b40-490c-868f-d11052c537c3] Updating instance_info_cache with network_info: [] {{(pid=54394) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1012.484199] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 415fb59a-a9db-424d-b066-03ead9103131] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port edb0dbf0-1d33-43ed-9943-e807cf7fc4e4, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 415fb59a-a9db-424d-b066-03ead9103131 was re-scheduled: Binding failed for port edb0dbf0-1d33-43ed-9943-e807cf7fc4e4, please check neutron logs for more information.\n'] [ 1012.484769] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 1012.484998] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 415fb59a-a9db-424d-b066-03ead9103131.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 415fb59a-a9db-424d-b066-03ead9103131. [ 1012.485303] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 415fb59a-a9db-424d-b066-03ead9103131] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 415fb59a-a9db-424d-b066-03ead9103131. [ 1012.507663] nova-conductor[54393]: DEBUG nova.network.neutron [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 415fb59a-a9db-424d-b066-03ead9103131] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1012.524778] nova-conductor[54393]: DEBUG nova.network.neutron [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 415fb59a-a9db-424d-b066-03ead9103131] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1012.529338] nova-conductor[54393]: DEBUG nova.network.neutron [None req-84541890-1688-448b-98cf-0d9293282c96 tempest-ServerDiskConfigTestJSON-246503124 tempest-ServerDiskConfigTestJSON-246503124-project-member] [instance: 415fb59a-a9db-424d-b066-03ead9103131] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1012.972499] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Expecting reply to msg 770a1e0d62bb4c2482d480628c3609bd in queue reply_65c7891fb9ba47458ad860069679c1e0 [ 1013.094086] nova-conductor[54393]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 770a1e0d62bb4c2482d480628c3609bd [ 1013.477515] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Took 0.51 seconds to select destinations for 1 instance(s). {{(pid=54393) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:946}} [ 1013.490230] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1013.490879] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1013.491199] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1013.518568] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1013.519080] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1013.519397] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1013.519895] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1013.520226] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1013.520574] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1013.530162] nova-conductor[54393]: DEBUG nova.quota [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Getting quotas for project 3332aa3d846f417e9a53239cadac267f. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 1013.533509] nova-conductor[54393]: DEBUG nova.quota [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Getting quotas for user 8a5c84e9df8543279986786a0174f8d5 and project 3332aa3d846f417e9a53239cadac267f. Resources: {'ram', 'instances', 'cores'} {{(pid=54393) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 1013.538366] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] [instance: 2836ea1d-893a-4c2e-8107-1ca23f59de62] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=54393) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1762}} [ 1013.538911] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1013.539227] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1013.539538] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1013.542649] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] [instance: 2836ea1d-893a-4c2e-8107-1ca23f59de62] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='756cebd3-6255-4a16-b83d-6f375a091ef0',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=54393) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1507}} [ 1013.543434] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1013.543773] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1013.544078] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1013.561325] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Acquiring lock "a4379ae1-47d8-4924-9d8e-3d761d765452" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1013.561742] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1013.562139] nova-conductor[54393]: DEBUG oslo_concurrency.lockutils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Lock "a4379ae1-47d8-4924-9d8e-3d761d765452" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=54393) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1023.525337] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell0:SELECT=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1023.546387] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:SELECT=3,nova_cell1:INSERT=7 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1023.557281] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=8 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1023.560177] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_api:SELECT=4,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1026.268890] nova-conductor[54393]: ERROR nova.scheduler.utils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] [instance: 2836ea1d-893a-4c2e-8107-1ca23f59de62] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2645, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 225, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 131, in wait\n current.throw(*self._exc)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 265, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2013, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1991, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 0e65a608-9ff4-4260-961d-f9fbe1c34d7f, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2460, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2752, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 2836ea1d-893a-4c2e-8107-1ca23f59de62 was re-scheduled: Binding failed for port 0e65a608-9ff4-4260-961d-f9fbe1c34d7f, please check neutron logs for more information.\n'] [ 1026.270868] nova-conductor[54393]: DEBUG nova.conductor.manager [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Rescheduling: True {{(pid=54393) build_instances /opt/stack/nova/nova/conductor/manager.py:696}} [ 1026.271286] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 2836ea1d-893a-4c2e-8107-1ca23f59de62.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 2836ea1d-893a-4c2e-8107-1ca23f59de62. [ 1026.271815] nova-conductor[54393]: WARNING nova.scheduler.utils [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] [instance: 2836ea1d-893a-4c2e-8107-1ca23f59de62] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 2836ea1d-893a-4c2e-8107-1ca23f59de62. [ 1026.299694] nova-conductor[54393]: DEBUG nova.network.neutron [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] [instance: 2836ea1d-893a-4c2e-8107-1ca23f59de62] deallocate_for_instance() {{(pid=54393) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1026.315948] nova-conductor[54393]: DEBUG nova.network.neutron [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] [instance: 2836ea1d-893a-4c2e-8107-1ca23f59de62] Instance cache missing network info. {{(pid=54393) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1026.318842] nova-conductor[54393]: DEBUG nova.network.neutron [None req-976cdf28-7f56-4b9a-9f74-8b714c36970d tempest-AttachVolumeTestJSON-603914290 tempest-AttachVolumeTestJSON-603914290-project-member] [instance: 2836ea1d-893a-4c2e-8107-1ca23f59de62] Updating instance_info_cache with network_info: [] {{(pid=54393) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1036.275570] nova-conductor[54394]: DEBUG dbcounter [-] [54394] Writing DB stats nova_cell1:SELECT=88,nova_cell1:UPDATE=41,nova_cell1:INSERT=6,nova_cell1:SAVEPOINT=10,nova_cell1:RELEASE=10 {{(pid=54394) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1036.328813] nova-conductor[54393]: DEBUG dbcounter [-] [54393] Writing DB stats nova_cell1:UPDATE=37,nova_cell1:SELECT=69,nova_cell1:SAVEPOINT=8,nova_cell1:RELEASE=8,nova_cell1:INSERT=4 {{(pid=54393) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}}