[ 486.648455] nova-conductor[53472]: INFO dbcounter [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] Registered counter for database nova_api [ 486.729309] nova-conductor[53472]: DEBUG oslo_db.sqlalchemy.engines [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=53472) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 486.732487] nova-conductor[53472]: DEBUG dbcounter [-] [53472] Writer thread running {{(pid=53472) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 486.754829] nova-conductor[53472]: DEBUG nova.context [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] Found 2 cells: 00000000-0000-0000-0000-000000000000(cell0),bfcf983b-0431-4fe7-9afc-0efa54598587(cell1) {{(pid=53472) load_cells /opt/stack/nova/nova/context.py:464}} [ 486.756615] nova-conductor[53472]: DEBUG oslo_concurrency.lockutils [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53472) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 486.756816] nova-conductor[53472]: DEBUG oslo_concurrency.lockutils [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53472) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 486.757257] nova-conductor[53472]: DEBUG oslo_concurrency.lockutils [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53472) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 486.757653] nova-conductor[53472]: DEBUG oslo_concurrency.lockutils [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53472) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 486.757839] nova-conductor[53472]: DEBUG oslo_concurrency.lockutils [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53472) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 486.758802] nova-conductor[53472]: DEBUG oslo_concurrency.lockutils [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53472) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 486.760095] nova-conductor[53472]: INFO dbcounter [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] Registered counter for database nova_cell0 [ 486.762319] nova-conductor[53472]: INFO dbcounter [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] Registered counter for database nova_cell1 [ 486.765041] nova-conductor[53472]: DEBUG oslo_db.sqlalchemy.engines [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=53472) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 486.765384] nova-conductor[53472]: DEBUG oslo_db.sqlalchemy.engines [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=53472) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 486.768309] nova-conductor[53472]: DEBUG dbcounter [-] [53472] Writer thread running {{(pid=53472) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 486.769120] nova-conductor[53472]: DEBUG dbcounter [-] [53472] Writer thread running {{(pid=53472) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 486.829513] nova-conductor[53472]: DEBUG oslo_concurrency.lockutils [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] Acquiring lock "singleton_lock" {{(pid=53472) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 486.829697] nova-conductor[53472]: DEBUG oslo_concurrency.lockutils [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] Acquired lock "singleton_lock" {{(pid=53472) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 486.829931] nova-conductor[53472]: DEBUG oslo_concurrency.lockutils [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] Releasing lock "singleton_lock" {{(pid=53472) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 486.830344] nova-conductor[53472]: INFO oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] Starting 2 workers [ 486.836231] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] Started child 53893 {{(pid=53472) _start_child /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/service.py:579}} [ 486.837987] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] Started child 53894 {{(pid=53472) _start_child /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/service.py:579}} [ 486.838661] nova-conductor[53893]: INFO nova.service [-] Starting conductor node (version 0.1.0) [ 486.839155] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] Full set of CONF: {{(pid=53472) wait /opt/stack/data/venv/lib/python3.10/site-packages/oslo_service/service.py:653}} [ 486.839356] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ******************************************************************************** {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2600}} [ 486.839508] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] Configuration options gathered from: {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2601}} [ 486.839716] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] command line args: ['--config-file', '/etc/nova/nova.conf'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2602}} [ 486.840453] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] config files: ['/etc/nova/nova.conf'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2603}} [ 486.840559] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ================================================================================ {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2605}} [ 486.840936] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] allow_resize_to_same_host = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.841302] nova-conductor[53894]: INFO nova.service [-] Starting conductor node (version 0.1.0) [ 486.841526] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] arq_binding_timeout = 300 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.841671] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] block_device_allocate_retries = 60 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.841868] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] block_device_allocate_retries_interval = 3 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.842092] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cert = self.pem {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.842284] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] compute_driver = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.842473] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] compute_monitors = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.842736] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] config_dir = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.842940] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] config_drive_format = iso9660 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.843088] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] config_file = ['/etc/nova/nova.conf'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.843278] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] config_source = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.843462] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] console_host = devstack {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.843658] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] control_exchange = nova {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.843888] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cpu_allocation_ratio = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.844082] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] daemon = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.844269] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] debug = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.844739] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] default_access_ip_network_name = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.844739] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] default_availability_zone = nova {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.844841] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] default_ephemeral_format = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.844947] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] default_green_pool_size = 1000 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.845239] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] default_log_levels = ['amqp=WARN', 'amqplib=WARN', 'boto=WARN', 'qpid=WARN', 'sqlalchemy=WARN', 'suds=INFO', 'oslo.messaging=INFO', 'oslo_messaging=INFO', 'iso8601=WARN', 'requests.packages.urllib3.connectionpool=WARN', 'urllib3.connectionpool=WARN', 'websocket=WARN', 'requests.packages.urllib3.util.retry=WARN', 'urllib3.util.retry=WARN', 'keystonemiddleware=WARN', 'routes.middleware=WARN', 'stevedore=WARN', 'taskflow=WARN', 'keystoneauth=WARN', 'oslo.cache=INFO', 'oslo_policy=INFO', 'dogpile.core.dogpile=INFO', 'glanceclient=WARN', 'oslo.privsep.daemon=INFO'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.845726] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] default_schedule_zone = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.845726] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] disk_allocation_ratio = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.845850] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] enable_new_services = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.846012] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] enabled_apis = ['osapi_compute'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.846212] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] enabled_ssl_apis = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.846520] nova-conductor[53894]: INFO dbcounter [None req-b96fba18-42e9-42ec-ab31-c8f44f71f971 None None] Registered counter for database nova_cell1 [ 486.846679] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] flat_injected = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.846874] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] force_config_drive = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.847065] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] force_raw_images = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.847238] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] graceful_shutdown_timeout = 5 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.849015] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] heal_instance_info_cache_interval = 60 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.849084] nova-conductor[53893]: INFO dbcounter [None req-27b68e8f-668d-445a-837e-1a877413108f None None] Registered counter for database nova_cell1 [ 486.849123] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] host = devstack {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.849123] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] initial_cpu_allocation_ratio = 4.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.849123] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] initial_disk_allocation_ratio = 1.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.849123] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] initial_ram_allocation_ratio = 1.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.849123] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] injected_network_template = /opt/stack/nova/nova/virt/interfaces.template {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.849123] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] instance_build_timeout = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.849300] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] instance_delete_interval = 300 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.849300] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] instance_format = [instance: %(uuid)s] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.849454] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] instance_name_template = instance-%08x {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.849623] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] instance_usage_audit = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.849832] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] instance_usage_audit_period = month {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.850024] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] instance_uuid_format = [instance: %(uuid)s] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.850218] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] instances_path = /opt/stack/data/nova/instances {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.850389] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] internal_service_availability_zone = internal {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.850545] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] key = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.850717] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] live_migration_retry_count = 30 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.850898] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] log_config_append = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.851122] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] log_date_format = %Y-%m-%d %H:%M:%S {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.851301] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] log_dir = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.851455] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] log_file = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.851578] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] log_options = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.851789] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] log_rotate_interval = 1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.851992] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] log_rotate_interval_type = days {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.852194] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] log_rotation_type = none {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.852399] nova-conductor[53894]: DEBUG oslo_db.sqlalchemy.engines [None req-b96fba18-42e9-42ec-ab31-c8f44f71f971 None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=53894) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 486.852541] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] logging_context_format_string = %(color)s%(levelname)s %(name)s [%(global_request_id)s %(request_id)s %(project_name)s %(user_name)s%(color)s] %(instance)s%(color)s%(message)s {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.852684] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] logging_debug_format_suffix = {{(pid=%(process)d) %(funcName)s %(pathname)s:%(lineno)d}} {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.852877] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] logging_default_format_string = %(color)s%(levelname)s %(name)s [-%(color)s] %(instance)s%(color)s%(message)s {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.853098] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] logging_exception_prefix = ERROR %(name)s %(instance)s {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.853239] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] logging_user_identity_format = %(user)s %(project)s %(domain)s %(system_scope)s %(user_domain)s %(project_domain)s {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.853478] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] long_rpc_timeout = 1800 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.853591] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] max_concurrent_builds = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.853763] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] max_concurrent_live_migrations = 1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.853934] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] max_concurrent_snapshots = 5 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.854103] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] max_local_block_devices = 3 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.854565] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] max_logfile_count = 30 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.854565] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] max_logfile_size_mb = 200 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.854565] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] maximum_instance_delete_attempts = 5 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.854733] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] metadata_listen = 0.0.0.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.854951] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] metadata_listen_port = 8775 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.855148] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] metadata_workers = 2 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.855332] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] migrate_max_retries = -1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.855492] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] mkisofs_cmd = genisoimage {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.855694] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] my_block_storage_ip = 10.180.1.21 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.855848] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] my_ip = 10.180.1.21 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.856552] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] network_allocate_retries = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.856552] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] non_inheritable_image_properties = ['cache_in_nova', 'bittorrent'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.856706] nova-conductor[53893]: DEBUG oslo_db.sqlalchemy.engines [None req-27b68e8f-668d-445a-837e-1a877413108f None None] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=53893) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 486.856775] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] osapi_compute_listen = 0.0.0.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.856775] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] osapi_compute_listen_port = 8774 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.856923] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] osapi_compute_unique_server_name_scope = {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.857113] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] osapi_compute_workers = 2 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.857274] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] password_length = 12 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.857449] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] periodic_enable = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.857603] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] periodic_fuzzy_delay = 60 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.857768] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] pointer_model = usbtablet {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.857968] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] preallocate_images = none {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.858134] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] publish_errors = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.858259] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] pybasedir = /opt/stack/nova {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.858700] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ram_allocation_ratio = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.858700] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] rate_limit_burst = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.858834] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] rate_limit_except_level = CRITICAL {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.859025] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] rate_limit_interval = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.859186] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] reboot_timeout = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.859359] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] reclaim_instance_interval = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.859515] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] record = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.859694] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] reimage_timeout_per_gb = 20 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.859888] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] report_interval = 120 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.860060] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] rescue_timeout = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.860237] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] reserved_host_cpus = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.860411] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] reserved_host_disk_mb = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.860566] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] reserved_host_memory_mb = 512 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.860750] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] reserved_huge_pages = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.860921] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] resize_confirm_window = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.861115] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] resize_fs_using_block_device = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.861289] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] resume_guests_state_on_host_boot = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.862024] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] rootwrap_config = /etc/nova/rootwrap.conf {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.862024] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] rpc_response_timeout = 60 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.862024] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] run_external_periodic_tasks = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.862024] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] running_deleted_instance_action = reap {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.862228] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] running_deleted_instance_poll_interval = 1800 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.862321] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] running_deleted_instance_timeout = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.862474] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] scheduler_instance_sync_interval = 120 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.862641] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] service_down_time = 720 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.862832] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] servicegroup_driver = db {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.862988] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] shelved_offload_time = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.863156] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] shelved_poll_interval = 3600 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.863321] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] shutdown_timeout = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.863476] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] source_is_ipv6 = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.863659] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writer thread running {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 486.863804] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ssl_only = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.863993] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] state_path = /opt/stack/data/nova {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.864169] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] sync_power_state_interval = 600 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.864353] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] sync_power_state_pool_size = 1000 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.864516] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] syslog_log_facility = LOG_USER {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.864670] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] tempdir = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.864836] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] timeout_nbd = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.865082] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] transport_url = **** {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.865253] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] update_resources_interval = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.865409] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] use_cow_images = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.865566] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] use_eventlog = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.865720] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] use_journal = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.865875] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] use_json = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.866065] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] use_rootwrap_daemon = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.866231] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] use_stderr = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.866393] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] use_syslog = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.866542] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vcpu_pin_set = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.866721] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vif_plugging_is_fatal = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.868062] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vif_plugging_timeout = 300 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.868062] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] virt_mkfs = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.868062] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] volume_usage_poll_interval = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.868062] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] watch_log_file = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.868062] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] web = /usr/share/spice-html5 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2613}} [ 486.868279] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_concurrency.disable_process_locking = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.868451] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_concurrency.lock_path = /opt/stack/data/nova {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.868669] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_metrics.metrics_buffer_size = 1000 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.868844] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_metrics.metrics_enabled = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.869023] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_metrics.metrics_process_name = {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.869199] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_metrics.metrics_socket_file = /var/tmp/metrics_collector.sock {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.869363] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_metrics.metrics_thread_stop_timeout = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.869584] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.auth_strategy = keystone {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.869784] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.compute_link_prefix = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.869990] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.config_drive_skip_versions = 1.0 2007-01-19 2007-03-01 2007-08-29 2007-10-10 2007-12-15 2008-02-01 2008-09-01 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.870179] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.dhcp_domain = novalocal {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.870347] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.enable_instance_password = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.870506] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.glance_link_prefix = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.870682] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.instance_list_cells_batch_fixed_size = 100 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.870875] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.instance_list_cells_batch_strategy = distributed {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.871051] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.instance_list_per_project_cells = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.871212] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.list_records_by_skipping_down_cells = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.871369] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.local_metadata_per_cell = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.871536] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.max_limit = 1000 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.871698] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.metadata_cache_expiration = 15 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.871867] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.neutron_default_tenant_id = default {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.872039] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.use_neutron_default_nets = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.872206] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.vendordata_dynamic_connect_timeout = 5 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.872367] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.vendordata_dynamic_failure_fatal = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.872551] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.vendordata_dynamic_read_timeout = 5 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.872720] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.vendordata_dynamic_ssl_certfile = {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.874018] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.vendordata_dynamic_targets = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.874018] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.vendordata_jsonfile_path = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.874018] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api.vendordata_providers = ['StaticJSON'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.874018] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.backend = dogpile.cache.memcached {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.874018] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.backend_argument = **** {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.874018] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.config_prefix = cache.oslo {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.874420] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writer thread running {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 486.874664] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.dead_timeout = 60.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.874856] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.debug_cache_backend = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.875056] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.enable_retry_client = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.875547] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.enable_socket_keepalive = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.875547] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.enabled = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.875547] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.enforce_fips_mode = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.875750] nova-conductor[53894]: DEBUG nova.service [None req-b96fba18-42e9-42ec-ab31-c8f44f71f971 None None] Creating RPC server for service conductor {{(pid=53894) start /opt/stack/nova/nova/service.py:182}} [ 486.875894] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.expiration_time = 600 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.876087] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.hashclient_retry_attempts = 2 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.876255] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.hashclient_retry_delay = 1.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.876415] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.memcache_dead_retry = 300 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.876573] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.memcache_password = **** {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.876781] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.memcache_pool_connection_get_timeout = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.879045] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.memcache_pool_flush_on_reconnect = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.879045] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.memcache_pool_maxsize = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.879045] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.memcache_pool_unused_timeout = 60 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.879045] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.memcache_sasl_enabled = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.879045] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.memcache_servers = ['localhost:11211'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.879045] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.memcache_socket_timeout = 1.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.879253] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.memcache_username = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.879287] nova-conductor[53893]: DEBUG nova.service [None req-27b68e8f-668d-445a-837e-1a877413108f None None] Creating RPC server for service conductor {{(pid=53893) start /opt/stack/nova/nova/service.py:182}} [ 486.879321] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.proxies = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.879321] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.redis_password = **** {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.879321] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.redis_sentinel_service_name = mymaster {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.879321] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.redis_sentinels = ['localhost:26379'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.879321] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.redis_server = localhost:6379 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.879321] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.redis_socket_timeout = 1.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.879491] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.redis_username = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.879521] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.retry_attempts = 2 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.879678] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.retry_delay = 0.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.879846] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.socket_keepalive_count = 1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.880010] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.socket_keepalive_idle = 1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.880177] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.socket_keepalive_interval = 1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.880626] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.tls_allowed_ciphers = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.880626] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.tls_cafile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.880711] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.tls_certfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.880792] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.tls_enabled = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.880945] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cache.tls_keyfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.881179] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cinder.auth_section = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.881358] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cinder.auth_type = password {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.881519] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cinder.cafile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.881713] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cinder.catalog_info = volumev3::publicURL {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.881870] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cinder.certfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.882039] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cinder.collect_timing = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.882217] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cinder.cross_az_attach = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.882394] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cinder.debug = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.882564] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cinder.endpoint_template = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.882739] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cinder.http_retries = 3 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.882897] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cinder.insecure = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.883058] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cinder.keyfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.883230] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cinder.os_region_name = RegionOne {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.883389] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cinder.split_loggers = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.883553] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cinder.timeout = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.883727] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] compute.consecutive_build_service_disable_threshold = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.883882] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] compute.cpu_dedicated_set = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.884046] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] compute.cpu_shared_set = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.884215] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] compute.image_type_exclude_list = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.884372] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] compute.live_migration_wait_for_vif_plug = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.884529] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] compute.max_concurrent_disk_ops = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.884685] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] compute.max_disk_devices_to_attach = -1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.884845] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] compute.packing_host_numa_cells_allocation_strategy = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.885022] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] compute.provider_config_location = /etc/nova/provider_config/ {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.885183] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] compute.resource_provider_association_refresh = 300 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.885343] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] compute.shutdown_retry_interval = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.885519] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] compute.vmdk_allowed_types = ['streamOptimized', 'monolithicSparse'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.885713] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] conductor.workers = 2 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.885893] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] console.allowed_origins = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.886066] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] console.ssl_ciphers = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.886236] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] console.ssl_minimum_version = default {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.886399] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] consoleauth.enforce_session_timeout = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.886560] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] consoleauth.token_ttl = 600 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.886788] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.cafile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.886947] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.certfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.887126] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.collect_timing = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.887305] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.connect_retries = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.887501] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.connect_retry_delay = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.887663] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.endpoint_override = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.887822] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.insecure = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.887975] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.keyfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.888141] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.max_version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.888301] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.min_version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.888451] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.region_name = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.888610] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.retriable_status_codes = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.888816] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.service_name = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.888999] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.service_type = accelerator {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.889175] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.split_loggers = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.889330] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.status_code_retries = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.889483] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.status_code_retry_delay = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.889661] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.timeout = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.889855] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.valid_interfaces = ['internal', 'public'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.890027] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] cyborg.version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.890217] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.backend = sqlalchemy {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.890388] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.connection = **** {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.890553] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.connection_debug = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.890723] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.connection_parameters = {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.890884] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.connection_recycle_time = 3600 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.891334] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.connection_trace = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.891334] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.db_inc_retry_interval = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.891399] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.db_max_retries = 20 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.891554] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.db_max_retry_interval = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.891711] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.db_retry_interval = 1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.891867] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.max_overflow = 50 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.892029] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.max_pool_size = 5 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.892188] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.max_retries = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.892352] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.mysql_sql_mode = TRADITIONAL {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.892504] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.mysql_wsrep_sync_wait = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.892666] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.pool_timeout = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.892821] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.retry_interval = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.892975] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.slave_connection = **** {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.893141] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.sqlite_synchronous = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.893301] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] database.use_db_reconnect = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.893472] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api_database.backend = sqlalchemy {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.893639] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api_database.connection = **** {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.893797] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api_database.connection_debug = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.893957] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api_database.connection_parameters = {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.894126] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api_database.connection_recycle_time = 3600 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.894285] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api_database.connection_trace = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.894439] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api_database.db_inc_retry_interval = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.894595] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api_database.db_max_retries = 20 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.894750] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api_database.db_max_retry_interval = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.894909] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api_database.db_retry_interval = 1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.895072] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api_database.max_overflow = 50 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.895226] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api_database.max_pool_size = 5 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.895380] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api_database.max_retries = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.895540] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api_database.mysql_sql_mode = TRADITIONAL {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.895699] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api_database.mysql_wsrep_sync_wait = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.895848] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api_database.pool_timeout = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.896015] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api_database.retry_interval = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.896163] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api_database.slave_connection = **** {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.896318] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] api_database.sqlite_synchronous = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.896507] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] devices.enabled_mdev_types = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.896734] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ephemeral_storage_encryption.cipher = aes-xts-plain64 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.896914] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ephemeral_storage_encryption.default_format = luks {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.897085] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ephemeral_storage_encryption.enabled = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.897426] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ephemeral_storage_encryption.key_size = 512 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.897426] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.api_servers = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.897603] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.cafile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.898016] nova-conductor[53894]: INFO oslo.messaging._drivers.impl_rabbit [None req-b96fba18-42e9-42ec-ab31-c8f44f71f971 None None] Creating fanout queue: conductor_fanout_eb183fb3e4f54cfb8487bbca2d0c580c [ 486.898158] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.certfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.898341] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.collect_timing = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.898529] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.connect_retries = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.898740] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.connect_retry_delay = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.899461] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.debug = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.899461] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.default_trusted_certificate_ids = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.899461] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.enable_certificate_validation = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.899461] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.enable_rbd_download = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.899820] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.endpoint_override = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.899820] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.insecure = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.899973] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.keyfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.900157] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.max_version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.900298] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.min_version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.900458] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.num_retries = 3 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.900628] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.rbd_ceph_conf = {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.900843] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.rbd_connect_timeout = 5 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.901050] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.rbd_pool = {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.901263] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.rbd_user = {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.901387] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.region_name = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.901545] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.retriable_status_codes = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.901851] nova-conductor[53893]: INFO oslo.messaging._drivers.impl_rabbit [None req-27b68e8f-668d-445a-837e-1a877413108f None None] Creating fanout queue: conductor_fanout_502477d22c624288a504fd6684a19da9 [ 486.902084] nova-conductor[53894]: DEBUG nova.service [None req-b96fba18-42e9-42ec-ab31-c8f44f71f971 None None] Join ServiceGroup membership for this service conductor {{(pid=53894) start /opt/stack/nova/nova/service.py:199}} [ 486.902231] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.service_name = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.902429] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.service_type = image {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.902605] nova-conductor[53894]: DEBUG nova.servicegroup.drivers.db [None req-b96fba18-42e9-42ec-ab31-c8f44f71f971 None None] DB_Driver: join new ServiceGroup member devstack to the conductor group, service = {{(pid=53894) join /opt/stack/nova/nova/servicegroup/drivers/db.py:44}} [ 486.902747] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.split_loggers = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.902924] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.status_code_retries = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.903105] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.status_code_retry_delay = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.903269] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.timeout = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.903454] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.valid_interfaces = ['internal', 'public'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.903621] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.verify_glance_signatures = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.903782] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] glance.version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.904009] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] guestfs.debug = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.904222] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] mks.enabled = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.904832] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] mks.mksproxy_base_url = http://127.0.0.1:6090/ {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.905062] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] image_cache.manager_interval = 2400 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.905266] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] image_cache.precache_concurrency = 1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.905439] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] image_cache.remove_unused_base_images = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.905628] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] image_cache.remove_unused_original_minimum_age_seconds = 86400 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.905850] nova-conductor[53893]: DEBUG nova.service [None req-27b68e8f-668d-445a-837e-1a877413108f None None] Join ServiceGroup membership for this service conductor {{(pid=53893) start /opt/stack/nova/nova/service.py:199}} [ 486.905995] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] image_cache.remove_unused_resized_minimum_age_seconds = 3600 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.906207] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] image_cache.subdirectory_name = _base {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.906393] nova-conductor[53893]: DEBUG nova.servicegroup.drivers.db [None req-27b68e8f-668d-445a-837e-1a877413108f None None] DB_Driver: join new ServiceGroup member devstack to the conductor group, service = {{(pid=53893) join /opt/stack/nova/nova/servicegroup/drivers/db.py:44}} [ 486.906539] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.api_max_retries = 60 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.906739] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.api_retry_interval = 2 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.906915] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.auth_section = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.907095] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.auth_type = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.907264] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.cafile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.907421] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.certfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.907585] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.collect_timing = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.907751] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.conductor_group = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.907937] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.connect_retries = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.908107] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.connect_retry_delay = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.908267] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.endpoint_override = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.908428] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.insecure = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.908583] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.keyfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.908769] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.max_version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.908931] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.min_version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.909109] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.peer_list = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.909269] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.region_name = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.909432] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.retriable_status_codes = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.909605] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.serial_console_state_timeout = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.909782] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.service_name = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.909974] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.service_type = baremetal {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.910150] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.shard = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.910318] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.split_loggers = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.910476] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.status_code_retries = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.910632] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.status_code_retry_delay = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.910791] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.timeout = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.910970] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.valid_interfaces = ['internal', 'public'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.911145] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ironic.version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.911350] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] key_manager.backend = nova.keymgr.conf_key_mgr.ConfKeyManager {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.911549] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] key_manager.fixed_key = **** {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.911772] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican.auth_endpoint = http://localhost/identity/v3 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.911961] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican.barbican_api_version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.912153] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican.barbican_endpoint = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.912355] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican.barbican_endpoint_type = public {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.912539] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican.barbican_region_name = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.912719] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican.cafile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.912918] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican.certfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.913112] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican.collect_timing = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.913282] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican.insecure = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.913441] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican.keyfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.913628] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican.number_of_retries = 60 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.913796] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican.retry_delay = 1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.913980] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican.send_service_user_token = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.914161] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican.split_loggers = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.914321] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican.timeout = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.914483] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican.verify_ssl = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.914665] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican.verify_ssl_path = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.914832] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican_service_user.auth_section = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.915044] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican_service_user.auth_type = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.915218] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican_service_user.cafile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.915377] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican_service_user.certfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.915542] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican_service_user.collect_timing = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.915702] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican_service_user.insecure = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.915862] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican_service_user.keyfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.916035] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican_service_user.split_loggers = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.916196] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] barbican_service_user.timeout = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.916366] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vault.approle_role_id = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.916525] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vault.approle_secret_id = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.916700] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vault.cafile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.916889] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vault.certfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.917066] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vault.collect_timing = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.917233] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vault.insecure = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.917388] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vault.keyfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.917564] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vault.kv_mountpoint = secret {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.917728] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vault.kv_path = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.917935] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vault.kv_version = 2 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.918119] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vault.namespace = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.918285] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vault.root_token_id = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.918449] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vault.split_loggers = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.918626] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vault.ssl_ca_crt_file = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.918797] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vault.timeout = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.918966] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vault.use_ssl = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.919172] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vault.vault_url = http://127.0.0.1:8200 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.919372] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.auth_section = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.919537] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.auth_type = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.919698] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.cafile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.919875] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.certfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.920050] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.collect_timing = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.920231] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.connect_retries = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.920388] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.connect_retry_delay = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.920542] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.endpoint_override = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.920703] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.insecure = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.920859] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.keyfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.921023] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.max_version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.921182] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.min_version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.921337] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.region_name = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.921492] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.retriable_status_codes = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.921645] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.service_name = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.921874] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.service_type = identity {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.922076] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.split_loggers = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.922246] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.status_code_retries = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.922416] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.status_code_retry_delay = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.922594] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.timeout = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.922783] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.valid_interfaces = ['internal', 'public'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.922946] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] keystone.version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.923210] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.connection_uri = {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.923392] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.cpu_mode = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.923565] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.cpu_model_extra_flags = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.923735] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.cpu_models = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.923927] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.cpu_power_governor_high = performance {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.924108] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.cpu_power_governor_low = powersave {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.924279] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.cpu_power_management = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.924451] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.cpu_power_management_strategy = cpu_state {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.924637] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.device_detach_attempts = 8 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.924820] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.device_detach_timeout = 20 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.925032] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.disk_cachemodes = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.925205] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.disk_prefix = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.925374] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.enabled_perf_events = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.925534] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.file_backed_memory = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.925698] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.gid_maps = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.925856] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.hw_disk_discard = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.926019] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.hw_machine_type = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.926191] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.images_rbd_ceph_conf = {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.926353] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.images_rbd_glance_copy_poll_interval = 15 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.926535] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.images_rbd_glance_copy_timeout = 600 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.926705] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.images_rbd_glance_store_name = {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.926873] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.images_rbd_pool = rbd {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.927072] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.images_type = default {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.927274] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.images_volume_group = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.927538] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.inject_key = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.927814] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.inject_partition = -2 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.928112] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.inject_password = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.928418] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.iscsi_iface = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.928700] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.iser_use_multipath = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.928904] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.live_migration_bandwidth = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.929093] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.live_migration_completion_timeout = 800 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.929267] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.live_migration_downtime = 500 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.929434] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.live_migration_downtime_delay = 75 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.929597] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.live_migration_downtime_steps = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.929766] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.live_migration_inbound_addr = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.929929] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.live_migration_permit_auto_converge = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.930103] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.live_migration_permit_post_copy = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.930284] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.live_migration_scheme = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.930455] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.live_migration_timeout_action = abort {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.930618] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.live_migration_tunnelled = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.930811] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.live_migration_uri = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.930985] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.live_migration_with_native_tls = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.931162] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.max_queues = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.931337] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.mem_stats_period_seconds = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.931686] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.migration_inbound_addr = 10.180.1.21 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.931880] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.nfs_mount_options = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.932121] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.nfs_mount_point_base = /opt/stack/data/nova/mnt {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.932313] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.num_aoe_discover_tries = 3 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.932479] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.num_iser_scan_tries = 5 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.932640] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.num_memory_encrypted_guests = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.932804] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.num_nvme_discover_tries = 5 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.932963] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.num_pcie_ports = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.933148] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.num_volume_scan_tries = 5 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.933317] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.pmem_namespaces = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.933478] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.quobyte_client_cfg = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.933701] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.quobyte_mount_point_base = /opt/stack/data/nova/mnt {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.933869] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.rbd_connect_timeout = 5 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.934043] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.rbd_destroy_volume_retries = 12 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.934210] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.rbd_destroy_volume_retry_interval = 5 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.934364] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.rbd_secret_uuid = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.934521] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.rbd_user = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.934683] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.realtime_scheduler_priority = 1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.934873] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.remote_filesystem_transport = ssh {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.935070] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.rescue_image_id = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.935234] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.rescue_kernel_id = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.935391] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.rescue_ramdisk_id = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.935561] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.rng_dev_path = /dev/urandom {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.935719] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.rx_queue_size = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.935891] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.smbfs_mount_options = {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.936119] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.smbfs_mount_point_base = /opt/stack/data/nova/mnt {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.936286] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.snapshot_compression = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.936444] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.snapshot_image_format = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.936686] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.snapshots_directory = /opt/stack/data/nova/instances/snapshots {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.936874] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.sparse_logical_volumes = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.937073] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.swtpm_enabled = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.937268] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.swtpm_group = tss {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.937446] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.swtpm_user = tss {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.937617] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.sysinfo_serial = unique {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.937776] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.tb_cache_size = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.937932] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.tx_queue_size = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.938158] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.uid_maps = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.938326] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.use_virtio_for_bridges = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.938494] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.virt_type = kvm {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.938688] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.volume_clear = zero {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.938862] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.volume_clear_size = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.939038] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.volume_use_multipath = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.939200] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.vzstorage_cache_path = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.939365] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.vzstorage_log_path = /var/log/vstorage/%(cluster_name)s/nova.log.gz {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.939529] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.vzstorage_mount_group = qemu {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.939716] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.vzstorage_mount_opts = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.939891] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.vzstorage_mount_perms = 0770 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.940120] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.vzstorage_mount_point_base = /opt/stack/data/nova/mnt {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.940297] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.vzstorage_mount_user = stack {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.940464] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] libvirt.wait_soft_reboot_seconds = 120 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.940659] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.auth_section = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.940835] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.auth_type = password {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.941025] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.cafile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.941191] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.certfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.941353] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.collect_timing = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.941508] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.connect_retries = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.941663] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.connect_retry_delay = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.941834] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.default_floating_pool = public {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.941992] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.endpoint_override = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.942170] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.extension_sync_interval = 600 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.942375] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.http_retries = 3 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.942499] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.insecure = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.942661] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.keyfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.942820] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.max_version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.942989] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.metadata_proxy_shared_secret = **** {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.943160] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.min_version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.943329] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.ovs_bridge = br-int {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.943493] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.physnets = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.943662] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.region_name = RegionOne {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.943824] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.retriable_status_codes = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.943992] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.service_metadata_proxy = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.944165] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.service_name = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.944338] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.service_type = network {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.944508] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.split_loggers = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.944665] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.status_code_retries = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.944824] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.status_code_retry_delay = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.944980] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.timeout = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.945179] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.valid_interfaces = ['internal', 'public'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.945342] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] neutron.version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.945514] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] notifications.bdms_in_notifications = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.945687] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] notifications.default_level = INFO {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.945861] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] notifications.notification_format = unversioned {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.946037] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] notifications.notify_on_state_change = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.946282] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] notifications.versioned_notifications_topics = ['versioned_notifications'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.946478] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] pci.alias = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.946672] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] pci.device_spec = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.946854] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] pci.report_in_placement = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.947081] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.auth_section = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.947270] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.auth_type = password {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.947471] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.auth_url = http://10.180.1.21/identity {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.947619] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.cafile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.947779] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.certfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.947939] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.collect_timing = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.948109] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.connect_retries = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.948271] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.connect_retry_delay = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.948427] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.default_domain_id = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.948582] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.default_domain_name = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.948789] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.domain_id = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.948962] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.domain_name = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.949137] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.endpoint_override = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.949300] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.insecure = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.949457] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.keyfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.949611] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.max_version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.949769] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.min_version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.949940] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.password = **** {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.950132] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.project_domain_id = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.950306] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.project_domain_name = Default {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.950470] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.project_id = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.950641] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.project_name = service {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.950819] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.region_name = RegionOne {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.951043] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.retriable_status_codes = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.951220] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.service_name = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.951393] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.service_type = placement {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.951557] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.split_loggers = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.951731] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.status_code_retries = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.951903] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.status_code_retry_delay = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.952075] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.system_scope = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.952236] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.timeout = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.952393] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.trust_id = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.952551] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.user_domain_id = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.952743] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.user_domain_name = Default {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.952905] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.user_id = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.953091] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.username = placement {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.953274] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.valid_interfaces = ['internal', 'public'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.953435] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] placement.version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.953620] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] quota.cores = 20 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.953788] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] quota.count_usage_from_placement = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.953959] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] quota.driver = nova.quota.DbQuotaDriver {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.954160] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] quota.injected_file_content_bytes = 10240 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.954328] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] quota.injected_file_path_length = 255 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.954492] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] quota.injected_files = 5 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.954658] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] quota.instances = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.954821] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] quota.key_pairs = 100 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.954981] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] quota.metadata_items = 128 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.955165] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] quota.ram = 51200 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.955327] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] quota.recheck_quota = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.955491] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] quota.server_group_members = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.955652] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] quota.server_groups = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.955852] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] scheduler.discover_hosts_in_cells_interval = -1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.956049] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] scheduler.enable_isolated_aggregate_filtering = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.956218] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] scheduler.image_metadata_prefilter = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.956398] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] scheduler.limit_tenants_to_placement_aggregate = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.956575] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] scheduler.max_attempts = 3 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.956762] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] scheduler.max_placement_results = 1000 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.956950] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] scheduler.placement_aggregate_required_for_tenants = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.957125] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] scheduler.query_placement_for_image_type_support = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.957303] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] scheduler.query_placement_for_routed_network_aggregates = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.957501] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] scheduler.workers = 2 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.957696] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.aggregate_image_properties_isolation_namespace = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.957868] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.aggregate_image_properties_isolation_separator = . {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.958056] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.available_filters = ['nova.scheduler.filters.all_filters'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.958226] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.build_failure_weight_multiplier = 1000000.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.958393] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.cpu_weight_multiplier = 1.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.958555] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.cross_cell_move_weight_multiplier = 1000000.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.958752] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.disk_weight_multiplier = 1.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.958972] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.enabled_filters = ['ComputeFilter', 'ComputeCapabilitiesFilter', 'ImagePropertiesFilter', 'ServerGroupAntiAffinityFilter', 'ServerGroupAffinityFilter', 'SameHostFilter', 'DifferentHostFilter'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.959158] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.host_subset_size = 1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.959324] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.hypervisor_version_weight_multiplier = 1.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.959488] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.image_properties_default_architecture = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.959649] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.io_ops_weight_multiplier = -1.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.959815] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.isolated_hosts = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.959981] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.isolated_images = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.960156] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.max_instances_per_host = 50 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.960336] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.max_io_ops_per_host = 8 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.960502] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.num_instances_weight_multiplier = 0.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.960668] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.pci_in_placement = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.960843] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.pci_weight_multiplier = 1.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.961014] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.ram_weight_multiplier = 1.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.961186] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.restrict_isolated_hosts_to_isolated_images = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.961350] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.shuffle_best_same_weighed_hosts = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.961509] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.soft_affinity_weight_multiplier = 1.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.961670] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.soft_anti_affinity_weight_multiplier = 1.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.961855] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.track_instance_changes = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.962045] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] filter_scheduler.weight_classes = ['nova.scheduler.weights.all_weighers'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.962218] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] metrics.required = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.962381] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] metrics.weight_multiplier = 1.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.962540] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] metrics.weight_of_unavailable = -10000.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.962701] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] metrics.weight_setting = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.963053] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] serial_console.base_url = ws://127.0.0.1:6083/ {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.963237] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] serial_console.enabled = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.963433] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] serial_console.port_range = 10000:20000 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.963607] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] serial_console.proxyclient_address = 127.0.0.1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.963778] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] serial_console.serialproxy_host = 0.0.0.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.963965] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] serial_console.serialproxy_port = 6083 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.964146] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] service_user.auth_section = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.964327] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] service_user.auth_type = password {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.964486] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] service_user.cafile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.964642] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] service_user.certfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.964828] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] service_user.collect_timing = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.964989] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] service_user.insecure = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.965153] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] service_user.keyfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.965323] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] service_user.send_service_user_token = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.965484] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] service_user.split_loggers = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.965639] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] service_user.timeout = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.965837] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] spice.agent_enabled = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.966007] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] spice.enabled = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.966352] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] spice.html5proxy_base_url = http://127.0.0.1:6082/spice_auto.html {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.966612] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] spice.html5proxy_host = 0.0.0.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.966796] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] spice.html5proxy_port = 6082 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.966957] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] spice.image_compression = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.967135] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] spice.jpeg_compression = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.967289] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] spice.playback_compression = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.967457] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] spice.server_listen = 127.0.0.1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.967624] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] spice.server_proxyclient_address = 127.0.0.1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.967781] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] spice.streaming_mode = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.967936] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] spice.zlib_compression = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.968142] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] upgrade_levels.baseapi = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.968314] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] upgrade_levels.compute = auto {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.968472] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] upgrade_levels.conductor = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.968653] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] upgrade_levels.scheduler = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.968847] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vendordata_dynamic_auth.auth_section = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.969024] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vendordata_dynamic_auth.auth_type = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.969189] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vendordata_dynamic_auth.cafile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.969349] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vendordata_dynamic_auth.certfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.969512] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vendordata_dynamic_auth.collect_timing = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.969728] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vendordata_dynamic_auth.insecure = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.969917] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vendordata_dynamic_auth.keyfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.970097] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vendordata_dynamic_auth.split_loggers = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.970258] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vendordata_dynamic_auth.timeout = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.970467] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.api_retry_count = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.970628] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.ca_file = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.970813] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.cache_prefix = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.970980] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.cluster_name = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.971151] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.connection_pool_size = 10 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.971308] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.console_delay_seconds = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.971464] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.datastore_regex = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.971622] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.host_ip = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.971779] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.host_password = **** {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.971938] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.host_port = 443 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.972109] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.host_username = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.972271] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.insecure = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.972426] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.integration_bridge = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.972584] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.maximum_objects = 100 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.972739] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.pbm_default_policy = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.972900] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.pbm_enabled = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.973064] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.pbm_wsdl_location = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.973232] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.serial_log_dir = /opt/vmware/vspc {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.973389] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.serial_port_proxy_uri = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.973542] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.serial_port_service_uri = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.973708] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.task_poll_interval = 0.5 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.973871] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.use_linked_clone = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.974046] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.vnc_keymap = en-us {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.974209] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.vnc_port = 5900 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.974372] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vmware.vnc_port_total = 10000 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.974953] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vnc.auth_schemes = ['none'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.975142] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vnc.enabled = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.975459] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vnc.novncproxy_base_url = http://127.0.0.1:6080/vnc_auto.html {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.975646] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vnc.novncproxy_host = 0.0.0.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.975819] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vnc.novncproxy_port = 6080 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.975996] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vnc.server_listen = 127.0.0.1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.976186] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vnc.server_proxyclient_address = 127.0.0.1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.976351] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vnc.vencrypt_ca_certs = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.976516] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vnc.vencrypt_client_cert = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.976688] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] vnc.vencrypt_client_key = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.976897] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.disable_compute_service_check_for_ffu = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.977075] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.disable_deep_image_inspection = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.977241] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.disable_fallback_pcpu_query = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.977400] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.disable_group_policy_check_upcall = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.977557] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.disable_libvirt_livesnapshot = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.977717] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.disable_rootwrap = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.977877] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.enable_numa_live_migration = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.978048] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.enable_qemu_monitor_announce_self = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.978211] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.ensure_libvirt_rbd_instance_dir_cleanup = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.978369] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.handle_virt_lifecycle_events = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.978526] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.libvirt_disable_apic = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.978711] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.never_download_image_if_on_rbd = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.978887] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.qemu_monitor_announce_self_count = 3 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.979061] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.qemu_monitor_announce_self_interval = 1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.979226] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.reserve_disk_resource_for_image_cache = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.979385] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.skip_cpu_compare_at_startup = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.979543] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.skip_cpu_compare_on_dest = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.979753] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.skip_hypervisor_version_check_on_lm = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.979921] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.skip_reserve_in_use_ironic_nodes = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.980095] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.unified_limits_count_pcpu_as_vcpu = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.980260] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] workarounds.wait_for_vif_plugged_event_during_hard_reboot = [] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.980465] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] wsgi.api_paste_config = /etc/nova/api-paste.ini {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.980638] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] wsgi.client_socket_timeout = 900 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.980807] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] wsgi.default_pool_size = 1000 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.980987] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] wsgi.keep_alive = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.981161] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] wsgi.max_header_line = 16384 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.981321] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] wsgi.secure_proxy_ssl_header = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.981479] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] wsgi.ssl_ca_file = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.981638] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] wsgi.ssl_cert_file = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.981800] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] wsgi.ssl_key_file = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.981961] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] wsgi.tcp_keepidle = 600 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.982175] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] wsgi.wsgi_log_format = %(client_ip)s "%(request_line)s" status: %(status_code)s len: %(body_length)s time: %(wall_seconds).7f {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.982356] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] zvm.ca_file = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.982515] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] zvm.cloud_connector_url = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.982738] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] zvm.image_tmp_path = /opt/stack/data/nova/images {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.982907] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] zvm.reachable_timeout = 300 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.983157] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_policy.enforce_new_defaults = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.983336] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_policy.enforce_scope = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.983511] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_policy.policy_default_rule = default {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.983688] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_policy.policy_dirs = ['policy.d'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.983858] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_policy.policy_file = policy.yaml {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.984062] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_policy.remote_content_type = application/x-www-form-urlencoded {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.984227] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_policy.remote_ssl_ca_crt_file = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.984383] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_policy.remote_ssl_client_crt_file = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.984539] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_policy.remote_ssl_client_key_file = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.984702] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_policy.remote_ssl_verify_server_crt = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.984893] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_versionedobjects.fatal_exception_format_errors = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.985090] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_middleware.http_basic_auth_user_file = /etc/htpasswd {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.985296] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] profiler.connection_string = messaging:// {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.985478] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] profiler.enabled = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.985663] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] profiler.es_doc_type = notification {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.985845] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] profiler.es_scroll_size = 10000 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.986018] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] profiler.es_scroll_time = 2m {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.986184] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] profiler.filter_error_trace = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.986349] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] profiler.hmac_keys = **** {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.986525] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] profiler.sentinel_service_name = mymaster {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.986706] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] profiler.socket_timeout = 0.1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.986870] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] profiler.trace_requests = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.987040] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] profiler.trace_sqlalchemy = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.987269] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] profiler_jaeger.process_tags = {} {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.987440] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] profiler_jaeger.service_name_prefix = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.987601] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] profiler_otlp.service_name_prefix = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.987802] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] remote_debug.host = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.987984] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] remote_debug.port = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.988192] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.amqp_auto_delete = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.988357] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.amqp_durable_queues = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.988524] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.conn_pool_min_size = 2 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.988732] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.conn_pool_ttl = 1200 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.988910] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.direct_mandatory_flag = True {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.989083] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.enable_cancel_on_failover = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.989251] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.heartbeat_in_pthread = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.989412] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.heartbeat_rate = 3 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.989570] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.heartbeat_timeout_threshold = 60 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.989763] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.hostname = devstack {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.989924] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.kombu_compression = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.990113] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.kombu_failover_strategy = round-robin {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.990281] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.kombu_missing_consumer_retry_timeout = 60 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.990452] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.kombu_reconnect_delay = 1.0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.990620] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.processname = nova-conductor {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.990786] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.rabbit_ha_queues = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.990961] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.rabbit_interval_max = 30 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.991147] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.rabbit_login_method = AMQPLAIN {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.991309] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.rabbit_qos_prefetch_count = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.991469] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.rabbit_quorum_delivery_limit = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.991628] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.rabbit_quorum_max_memory_bytes = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.991793] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.rabbit_quorum_max_memory_length = 0 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.991948] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.rabbit_quorum_queue = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.992121] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.rabbit_retry_backoff = 2 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.992280] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.rabbit_retry_interval = 1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.992432] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.rabbit_stream_fanout = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.992592] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.rabbit_transient_queues_ttl = 1800 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.992747] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.rabbit_transient_quorum_queue = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.992914] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.rpc_conn_pool_size = 30 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.993094] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.ssl = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.993268] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.ssl_ca_file = {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.993436] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.ssl_cert_file = {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.993596] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.ssl_enforce_fips_mode = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.993763] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.ssl_key_file = {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.993935] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.ssl_version = {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.994111] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_rabbit.use_queue_manager = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.994373] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_notifications.driver = ['messagingv2'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.994545] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_notifications.retry = -1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.994728] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_notifications.topics = ['notifications'] {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.994904] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_messaging_notifications.transport_url = **** {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.995115] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.auth_section = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.995289] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.auth_type = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.995447] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.cafile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.995605] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.certfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.995767] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.collect_timing = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.995923] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.connect_retries = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.996089] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.connect_retry_delay = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.996246] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.endpoint_id = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.996397] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.endpoint_override = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.996575] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.insecure = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.996739] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.keyfile = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.996901] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.max_version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.997072] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.min_version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.997232] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.region_name = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.997388] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.retriable_status_codes = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.997550] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.service_name = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.997701] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.service_type = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.997872] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.split_loggers = False {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.998036] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.status_code_retries = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.998196] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.status_code_retry_delay = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.998350] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.timeout = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.998506] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.valid_interfaces = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.998686] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_limit.version = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.998891] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_reports.file_event_handler = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.999096] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_reports.file_event_handler_interval = 1 {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.999310] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] oslo_reports.log_dir = None {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2620}} [ 486.999412] nova-conductor[53472]: DEBUG oslo_service.service [None req-46d2e805-11b1-4d1d-897f-babb7dd8a4bb None None] ******************************************************************************** {{(pid=53472) log_opt_values /opt/stack/data/venv/lib/python3.10/site-packages/oslo_config/cfg.py:2624}} [ 496.753556] nova-conductor[53472]: DEBUG dbcounter [-] [53472] Writing DB stats nova_api:SELECT=3 {{(pid=53472) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 496.825843] nova-conductor[53472]: DEBUG dbcounter [-] [53472] Writing DB stats nova_cell1:SELECT=3 {{(pid=53472) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 496.828415] nova-conductor[53472]: DEBUG dbcounter [-] [53472] Writing DB stats nova_cell0:SELECT=3 {{(pid=53472) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 501.913069] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=10,nova_cell1:INSERT=1,nova_cell1:UPDATE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 501.914648] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=7 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 569.132700] nova-conductor[53893]: DEBUG oslo_db.sqlalchemy.engines [None req-c011099e-515a-472e-a8f7-3f9256b6a434 None None] Parent process 53472 forked (53893) with an open database connection, which is being discarded and recreated. {{(pid=53893) checkout /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:438}} [ 569.135362] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writer thread running {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 569.135707] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=4 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 579.138294] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 579.863256] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=15,nova_cell1:INSERT=1,nova_cell1:UPDATE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 584.892901] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=15,nova_cell1:INSERT=1,nova_cell1:UPDATE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 612.664924] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Creating reply queue: reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 612.672314] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Expecting reply to msg c80c03305482455c9eabf3cf96b17909 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 613.112058] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c80c03305482455c9eabf3cf96b17909 [ 613.112058] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Took 0.45 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 613.146084] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.146387] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.148026] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.002s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.150211] nova-conductor[53893]: INFO dbcounter [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Registered counter for database nova_cell1 [ 613.154596] nova-conductor[53893]: DEBUG oslo_db.sqlalchemy.engines [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=53893) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 613.158340] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writer thread running {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 613.223649] nova-conductor[53894]: DEBUG oslo_db.sqlalchemy.engines [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Parent process 53472 forked (53894) with an open database connection, which is being discarded and recreated. {{(pid=53894) checkout /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:438}} [ 613.228133] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writer thread running {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 613.235300] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_api:SELECT=4 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 613.250878] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Creating reply queue: reply_221f6b1d765f476aaa22fe0a28260c97 [ 613.261351] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Expecting reply to msg 30e34068114b46e0b09d09a04c4bdae1 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 613.289166] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.291995] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.003s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.292595] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.293984] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.294768] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.295401] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.297086] nova-conductor[53893]: INFO dbcounter [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Registered counter for database nova_cell0 [ 613.307028] nova-conductor[53893]: DEBUG oslo_db.sqlalchemy.engines [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=53893) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 613.311549] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writer thread running {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 613.331588] nova-conductor[53893]: DEBUG nova.quota [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Getting quotas for project a36fff95014d4d868f65dafefe1958fe. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 613.337468] nova-conductor[53893]: DEBUG nova.quota [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Getting quotas for user 26298f53a4904c86a795c9a1717500e7 and project a36fff95014d4d868f65dafefe1958fe. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 613.347832] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] [instance: ef6e717f-c8b2-4536-8425-89f218d49f1a] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 613.348456] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.348696] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.349871] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.359138] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] [instance: ef6e717f-c8b2-4536-8425-89f218d49f1a] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 613.360014] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.360318] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.360396] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.398516] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.398836] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.398954] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.399332] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Acquiring lock "compute-rpcapi-router" {{(pid=53893) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 613.399435] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Acquired lock "compute-rpcapi-router" {{(pid=53893) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 613.399998] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ff1647e-8c8a-451f-aedb-15622758fd34 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.400156] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ff1647e-8c8a-451f-aedb-15622758fd34 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.400347] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ff1647e-8c8a-451f-aedb-15622758fd34 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.401230] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ff1647e-8c8a-451f-aedb-15622758fd34 None None] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.402689] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ff1647e-8c8a-451f-aedb-15622758fd34 None None] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.402689] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ff1647e-8c8a-451f-aedb-15622758fd34 None None] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.409672] nova-conductor[53893]: INFO nova.compute.rpcapi [None req-5ff1647e-8c8a-451f-aedb-15622758fd34 None None] Automatically selected compute RPC version 6.3 from minimum service version 67 [ 613.410205] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ff1647e-8c8a-451f-aedb-15622758fd34 None None] Releasing lock "compute-rpcapi-router" {{(pid=53893) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 613.511133] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 30e34068114b46e0b09d09a04c4bdae1 [ 613.511869] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Took 0.27 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 613.555569] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.555855] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.559189] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.559789] nova-conductor[53894]: INFO dbcounter [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Registered counter for database nova_cell1 [ 613.566737] nova-conductor[53894]: DEBUG oslo_db.sqlalchemy.engines [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=53894) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 613.569338] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writer thread running {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 613.636367] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.636591] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.637044] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.637518] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.637703] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.638742] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.640285] nova-conductor[53894]: INFO dbcounter [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Registered counter for database nova_cell0 [ 613.650365] nova-conductor[53894]: DEBUG oslo_db.sqlalchemy.engines [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] MySQL server mode set to STRICT_TRANS_TABLES,STRICT_ALL_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,TRADITIONAL,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION {{(pid=53894) _check_effective_sql_mode /opt/stack/data/venv/lib/python3.10/site-packages/oslo_db/sqlalchemy/engines.py:342}} [ 613.660031] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writer thread running {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:102}} [ 613.673456] nova-conductor[53894]: DEBUG nova.quota [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Getting quotas for project 82a8caeb29874fcd98cafe0ded9fe9b1. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 613.677401] nova-conductor[53894]: DEBUG nova.quota [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Getting quotas for user 32eab6d94a6e4ebfa275cafb59dccce5 and project 82a8caeb29874fcd98cafe0ded9fe9b1. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 613.683642] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] [instance: 04fd7039-c2c8-4b78-8c3d-37eb66fe2115] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 613.684236] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.684438] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.684599] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.708918] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] [instance: 04fd7039-c2c8-4b78-8c3d-37eb66fe2115] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 613.709421] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.709715] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.709910] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.753170] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.753470] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.754756] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.754756] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Acquiring lock "compute-rpcapi-router" {{(pid=53894) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:310}} [ 613.754756] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Acquired lock "compute-rpcapi-router" {{(pid=53894) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:313}} [ 613.754756] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-fe8a495b-5d0b-4c96-8c5d-be0288e5f9f2 None None] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.754982] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-fe8a495b-5d0b-4c96-8c5d-be0288e5f9f2 None None] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.754982] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-fe8a495b-5d0b-4c96-8c5d-be0288e5f9f2 None None] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.755353] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-fe8a495b-5d0b-4c96-8c5d-be0288e5f9f2 None None] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 613.755525] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-fe8a495b-5d0b-4c96-8c5d-be0288e5f9f2 None None] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 613.755703] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-fe8a495b-5d0b-4c96-8c5d-be0288e5f9f2 None None] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 613.764153] nova-conductor[53894]: INFO nova.compute.rpcapi [None req-fe8a495b-5d0b-4c96-8c5d-be0288e5f9f2 None None] Automatically selected compute RPC version 6.3 from minimum service version 67 [ 613.764437] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-fe8a495b-5d0b-4c96-8c5d-be0288e5f9f2 None None] Releasing lock "compute-rpcapi-router" {{(pid=53894) lock /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:331}} [ 618.745827] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Expecting reply to msg 9491fe5fbf6a4e86bc9954a49294e399 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 618.927221] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9491fe5fbf6a4e86bc9954a49294e399 [ 618.928147] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Took 0.19 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 618.947742] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 618.948570] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 618.948570] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 618.993398] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 618.993755] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 618.994301] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 618.994901] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 618.995096] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 618.995342] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 619.010985] nova-conductor[53893]: DEBUG nova.quota [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Getting quotas for project 8d235e06cce044529d4f982bf37af3e8. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 619.014166] nova-conductor[53893]: DEBUG nova.quota [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Getting quotas for user 34cf0f0eff664be3be9da8676c5bd742 and project 8d235e06cce044529d4f982bf37af3e8. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 619.021723] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: fd68bfa8-62af-490d-bfad-5ef59a733336] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 619.022298] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 619.022805] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 619.022805] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 619.030640] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: fd68bfa8-62af-490d-bfad-5ef59a733336] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 619.031376] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 619.031720] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 619.031805] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 619.059682] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 619.059890] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 619.060106] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 622.212379] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Expecting reply to msg f88196ae008f4ba8a1331567bb6ccda4 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 622.392458] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f88196ae008f4ba8a1331567bb6ccda4 [ 622.393300] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Took 0.18 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 622.416182] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 622.416182] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 622.416182] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 622.466155] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 622.466451] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 622.466850] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 622.467579] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 622.468284] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 622.468284] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 622.481302] nova-conductor[53894]: DEBUG nova.quota [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Getting quotas for project 8d235e06cce044529d4f982bf37af3e8. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 622.486666] nova-conductor[53894]: DEBUG nova.quota [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Getting quotas for user 34cf0f0eff664be3be9da8676c5bd742 and project 8d235e06cce044529d4f982bf37af3e8. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 622.493607] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: 73f9bdec-1446-4293-a1d9-cfa4ef2dd9dd] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 622.494269] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 622.494492] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 622.494660] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 622.500568] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: 73f9bdec-1446-4293-a1d9-cfa4ef2dd9dd] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 622.501334] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 622.501475] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 622.501671] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 622.523898] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 622.524270] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 622.524514] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 623.875982] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] [instance: ef6e717f-c8b2-4536-8425-89f218d49f1a] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 2b013813-1b56-4582-98f4-af17c7453ad5, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance ef6e717f-c8b2-4536-8425-89f218d49f1a was re-scheduled: Binding failed for port 2b013813-1b56-4582-98f4-af17c7453ad5, please check neutron logs for more information.\n'] [ 623.876695] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 623.877036] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ef6e717f-c8b2-4536-8425-89f218d49f1a.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ef6e717f-c8b2-4536-8425-89f218d49f1a. [ 623.877497] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] [instance: ef6e717f-c8b2-4536-8425-89f218d49f1a] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ef6e717f-c8b2-4536-8425-89f218d49f1a. [ 623.936267] nova-conductor[53894]: DEBUG nova.network.neutron [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] [instance: ef6e717f-c8b2-4536-8425-89f218d49f1a] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 624.108169] nova-conductor[53894]: DEBUG nova.network.neutron [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] [instance: ef6e717f-c8b2-4536-8425-89f218d49f1a] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 624.115976] nova-conductor[53894]: DEBUG nova.network.neutron [None req-a4584c8c-7c17-423e-bd49-7eab8ace72e3 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] [instance: ef6e717f-c8b2-4536-8425-89f218d49f1a] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 625.334893] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Expecting reply to msg 971e80ba8d5d4e2eae29027f56b2d0b6 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 625.535464] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 971e80ba8d5d4e2eae29027f56b2d0b6 [ 625.538703] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Took 0.21 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 625.557039] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 625.557277] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 625.557444] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 625.621130] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 625.621383] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 625.621561] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 625.621970] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 625.622228] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 625.622344] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 625.639820] nova-conductor[53893]: DEBUG nova.quota [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Getting quotas for project 889dea457e904a0c84b6837db10948a8. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 625.643485] nova-conductor[53893]: DEBUG nova.quota [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Getting quotas for user 2a6922084d9f439fa1118565e894e537 and project 889dea457e904a0c84b6837db10948a8. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 625.654488] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] [instance: 151fefe2-b70a-4ea5-8b50-08c7968b10fe] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 625.655015] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 625.658070] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 625.658070] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 625.661684] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] [instance: 151fefe2-b70a-4ea5-8b50-08c7968b10fe] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 625.662902] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 625.662902] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 625.662902] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 625.711387] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 625.711387] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 625.711387] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 627.431025] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: fd68bfa8-62af-490d-bfad-5ef59a733336] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port c28c4885-b80c-41e0-a036-b309a3352a9a, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance fd68bfa8-62af-490d-bfad-5ef59a733336 was re-scheduled: Binding failed for port c28c4885-b80c-41e0-a036-b309a3352a9a, please check neutron logs for more information.\n'] [ 627.432253] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 627.433703] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance fd68bfa8-62af-490d-bfad-5ef59a733336.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance fd68bfa8-62af-490d-bfad-5ef59a733336. [ 627.433703] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: fd68bfa8-62af-490d-bfad-5ef59a733336] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance fd68bfa8-62af-490d-bfad-5ef59a733336. [ 627.494055] nova-conductor[53893]: DEBUG nova.network.neutron [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: fd68bfa8-62af-490d-bfad-5ef59a733336] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 627.693425] nova-conductor[53893]: DEBUG nova.network.neutron [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: fd68bfa8-62af-490d-bfad-5ef59a733336] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 627.701037] nova-conductor[53893]: DEBUG nova.network.neutron [None req-37842a60-c148-4917-be75-8a9a7135040a tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: fd68bfa8-62af-490d-bfad-5ef59a733336] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 628.919080] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Expecting reply to msg dd1fe0cc321c48c99f1fb69198567fe6 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 629.051830] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: 73f9bdec-1446-4293-a1d9-cfa4ef2dd9dd] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 55260f69-7e9b-4733-95ed-eef39c8b95bf, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 73f9bdec-1446-4293-a1d9-cfa4ef2dd9dd was re-scheduled: Binding failed for port 55260f69-7e9b-4733-95ed-eef39c8b95bf, please check neutron logs for more information.\n'] [ 629.052550] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 629.053272] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 73f9bdec-1446-4293-a1d9-cfa4ef2dd9dd.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 73f9bdec-1446-4293-a1d9-cfa4ef2dd9dd. [ 629.053716] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: 73f9bdec-1446-4293-a1d9-cfa4ef2dd9dd] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 73f9bdec-1446-4293-a1d9-cfa4ef2dd9dd. [ 629.079020] nova-conductor[53893]: DEBUG nova.network.neutron [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: 73f9bdec-1446-4293-a1d9-cfa4ef2dd9dd] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 629.091493] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg dd1fe0cc321c48c99f1fb69198567fe6 [ 629.092361] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Took 0.18 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 629.109796] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 629.109966] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 629.110165] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 629.110526] nova-conductor[53893]: DEBUG nova.network.neutron [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: 73f9bdec-1446-4293-a1d9-cfa4ef2dd9dd] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 629.114601] nova-conductor[53893]: DEBUG nova.network.neutron [None req-ce5b7126-d3f0-4fe5-bc44-ac1395dd3fe3 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: 73f9bdec-1446-4293-a1d9-cfa4ef2dd9dd] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 629.150409] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 629.150650] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 629.150821] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 629.151296] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 629.151515] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 629.151700] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 629.161871] nova-conductor[53894]: DEBUG nova.quota [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Getting quotas for project 8d235e06cce044529d4f982bf37af3e8. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 629.164846] nova-conductor[53894]: DEBUG nova.quota [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Getting quotas for user 34cf0f0eff664be3be9da8676c5bd742 and project 8d235e06cce044529d4f982bf37af3e8. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 629.174652] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: 8e95b690-9fa8-4a15-901f-1bf318bfff10] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 629.177032] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 629.177032] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 629.177032] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 629.180738] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: 8e95b690-9fa8-4a15-901f-1bf318bfff10] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 629.181607] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 629.181908] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 629.182143] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 629.229456] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 629.229456] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 629.229456] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 632.532604] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Expecting reply to msg 8c9c170df49a465f8fac76a0de6b422e in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 632.713503] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8c9c170df49a465f8fac76a0de6b422e [ 632.713503] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Took 0.18 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 632.736995] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 632.737279] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 632.737462] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 632.812148] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 632.812389] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 632.812561] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 632.812939] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 632.813139] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 632.813297] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 632.832136] nova-conductor[53894]: DEBUG nova.quota [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Getting quotas for project d3952a0eb9b246b3981a76df98b855f0. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 632.836182] nova-conductor[53894]: DEBUG nova.quota [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Getting quotas for user 39dd18ce542f42e08215016404ffe9dc and project d3952a0eb9b246b3981a76df98b855f0. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 632.843960] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] [instance: 63bccd39-d951-4187-872d-559cd7fead30] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 632.844961] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 632.845258] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 632.845482] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 632.852251] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] [instance: 63bccd39-d951-4187-872d-559cd7fead30] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 632.852961] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 632.853189] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 632.853357] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 632.896403] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 632.896705] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 632.896843] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 633.405646] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: 8e95b690-9fa8-4a15-901f-1bf318bfff10] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 3ee45c30-d84c-487e-b9b1-efc497508ce9, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 8e95b690-9fa8-4a15-901f-1bf318bfff10 was re-scheduled: Binding failed for port 3ee45c30-d84c-487e-b9b1-efc497508ce9, please check neutron logs for more information.\n'] [ 633.406231] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 633.406455] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8e95b690-9fa8-4a15-901f-1bf318bfff10.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8e95b690-9fa8-4a15-901f-1bf318bfff10. [ 633.406659] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: 8e95b690-9fa8-4a15-901f-1bf318bfff10] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8e95b690-9fa8-4a15-901f-1bf318bfff10. [ 633.453236] nova-conductor[53893]: DEBUG nova.network.neutron [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: 8e95b690-9fa8-4a15-901f-1bf318bfff10] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 633.477439] nova-conductor[53893]: DEBUG nova.network.neutron [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: 8e95b690-9fa8-4a15-901f-1bf318bfff10] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 633.486266] nova-conductor[53893]: DEBUG nova.network.neutron [None req-1b45b4e9-2700-41dc-a982-68d4e2d91040 tempest-ListServerFiltersTestJSON-89530523 tempest-ListServerFiltersTestJSON-89530523-project-member] [instance: 8e95b690-9fa8-4a15-901f-1bf318bfff10] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 634.072016] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Expecting reply to msg 061949cd8bd04bd78c42acd2cb435fac in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 634.299181] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 061949cd8bd04bd78c42acd2cb435fac [ 634.300680] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Took 0.23 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 634.356569] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 634.356569] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 634.356569] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 634.437454] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 634.437752] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 634.437998] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 634.438450] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 634.438697] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 634.439289] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 634.456746] nova-conductor[53893]: DEBUG nova.quota [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Getting quotas for project b410a9b0a0a54c00bbbaa088cd81b957. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 634.461066] nova-conductor[53893]: DEBUG nova.quota [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Getting quotas for user a893b6f20a704d77968791ca48532894 and project b410a9b0a0a54c00bbbaa088cd81b957. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 634.472655] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] [instance: 98916e4a-561b-4c17-9903-de88c3678f13] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 634.473142] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 634.473347] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 634.473518] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 634.481348] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] [instance: 98916e4a-561b-4c17-9903-de88c3678f13] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 634.481762] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 634.482106] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 634.482585] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 634.520343] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 634.521060] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 634.521060] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 635.799890] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Expecting reply to msg 293608a1a9654c5b8e272f3cc1b5695f in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 636.012332] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 293608a1a9654c5b8e272f3cc1b5695f [ 636.013252] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Took 0.22 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 636.044840] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 636.044840] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 636.044840] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 636.095494] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 636.096772] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 636.096772] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 636.096772] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 636.096772] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 636.096772] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 636.111187] nova-conductor[53894]: DEBUG nova.quota [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Getting quotas for project 09f40d4f7f5f4191ab268f6dc55509c5. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 636.118610] nova-conductor[53894]: DEBUG nova.quota [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Getting quotas for user c55f4f7f38974ff6a84cabbb31fbf5d1 and project 09f40d4f7f5f4191ab268f6dc55509c5. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 636.126731] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] [instance: 0ce4707a-de75-438a-be72-d829478bfdff] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 636.126731] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 636.126731] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 636.126731] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 636.135549] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] [instance: 0ce4707a-de75-438a-be72-d829478bfdff] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 636.136268] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 636.136481] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 636.136652] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 636.161747] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 636.161987] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 636.162187] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 637.391599] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Expecting reply to msg 41159aa26ad9494598170ecb12b95eb9 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 637.542063] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 41159aa26ad9494598170ecb12b95eb9 [ 637.542728] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Took 0.15 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 637.559639] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 637.561486] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 637.561775] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.002s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 637.616139] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 637.616139] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 637.616139] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 637.616139] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 637.616139] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 637.616139] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 637.626756] nova-conductor[53893]: DEBUG nova.quota [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Getting quotas for project a36fff95014d4d868f65dafefe1958fe. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 637.630417] nova-conductor[53893]: DEBUG nova.quota [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Getting quotas for user 26298f53a4904c86a795c9a1717500e7 and project a36fff95014d4d868f65dafefe1958fe. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 637.636450] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] [instance: 00e11309-0f82-49d5-b4f5-02e2bdb517e8] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 637.637157] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 637.637499] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 637.637816] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 637.644920] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] [instance: 00e11309-0f82-49d5-b4f5-02e2bdb517e8] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 637.644920] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 637.644920] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 637.644920] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 637.669031] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 637.669031] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 637.669031] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 642.201664] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Expecting reply to msg f179be9670744a00b224855834c5b308 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 642.367944] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f179be9670744a00b224855834c5b308 [ 642.368971] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Took 0.17 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 642.385371] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 642.385677] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 642.385907] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 642.430564] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 642.430743] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 642.430917] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 642.431305] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 642.431492] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 642.431657] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 642.452815] nova-conductor[53894]: DEBUG nova.quota [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Getting quotas for project 2e2f2a89dde94e7ab986cfa80346c4e5. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 642.457644] nova-conductor[53894]: DEBUG nova.quota [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Getting quotas for user fec462edbe9546b587ce798c89d1cd3a and project 2e2f2a89dde94e7ab986cfa80346c4e5. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 642.468173] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] [instance: 1385849e-9e64-4062-b6aa-300e6e7eab3d] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 642.468699] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 642.468912] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 642.469342] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 642.474628] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] [instance: 1385849e-9e64-4062-b6aa-300e6e7eab3d] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 642.475301] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 642.475510] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 642.475680] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 642.497672] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 642.497898] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 642.498097] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 644.425792] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] [instance: 98916e4a-561b-4c17-9903-de88c3678f13] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 67790ac8-de66-4ed2-a07c-71ee46e65aa6, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 98916e4a-561b-4c17-9903-de88c3678f13 was re-scheduled: Binding failed for port 67790ac8-de66-4ed2-a07c-71ee46e65aa6, please check neutron logs for more information.\n'] [ 644.428080] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 644.428835] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 98916e4a-561b-4c17-9903-de88c3678f13.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 98916e4a-561b-4c17-9903-de88c3678f13. [ 644.432145] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] [instance: 98916e4a-561b-4c17-9903-de88c3678f13] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 98916e4a-561b-4c17-9903-de88c3678f13. [ 644.463114] nova-conductor[53894]: DEBUG nova.network.neutron [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] [instance: 98916e4a-561b-4c17-9903-de88c3678f13] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 644.525009] nova-conductor[53894]: DEBUG nova.network.neutron [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] [instance: 98916e4a-561b-4c17-9903-de88c3678f13] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 644.531732] nova-conductor[53894]: DEBUG nova.network.neutron [None req-da7f62a9-cb31-4f18-896e-526b55621fa7 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] [instance: 98916e4a-561b-4c17-9903-de88c3678f13] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 645.577419] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] [instance: 0ce4707a-de75-438a-be72-d829478bfdff] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 04789809-11c8-496c-ad05-374e7e031000, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 0ce4707a-de75-438a-be72-d829478bfdff was re-scheduled: Binding failed for port 04789809-11c8-496c-ad05-374e7e031000, please check neutron logs for more information.\n'] [ 645.577765] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 645.578320] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 0ce4707a-de75-438a-be72-d829478bfdff.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 0ce4707a-de75-438a-be72-d829478bfdff. [ 645.578625] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] [instance: 0ce4707a-de75-438a-be72-d829478bfdff] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 0ce4707a-de75-438a-be72-d829478bfdff. [ 645.608864] nova-conductor[53894]: DEBUG nova.network.neutron [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] [instance: 0ce4707a-de75-438a-be72-d829478bfdff] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 645.634603] nova-conductor[53894]: DEBUG nova.network.neutron [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] [instance: 0ce4707a-de75-438a-be72-d829478bfdff] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 645.641131] nova-conductor[53894]: DEBUG nova.network.neutron [None req-841ccec2-9ff5-4877-ba11-d94213a4e089 tempest-ServersAdminNegativeTestJSON-1310685096 tempest-ServersAdminNegativeTestJSON-1310685096-project-member] [instance: 0ce4707a-de75-438a-be72-d829478bfdff] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 645.937462] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Expecting reply to msg b79e4e0147144351b9ecbcfa32fc316e in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 645.962523] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] [instance: 00e11309-0f82-49d5-b4f5-02e2bdb517e8] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 2c49d0c6-a457-48eb-971f-541920c2e4f2, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 00e11309-0f82-49d5-b4f5-02e2bdb517e8 was re-scheduled: Binding failed for port 2c49d0c6-a457-48eb-971f-541920c2e4f2, please check neutron logs for more information.\n'] [ 646.169451] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 646.169451] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 00e11309-0f82-49d5-b4f5-02e2bdb517e8.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 00e11309-0f82-49d5-b4f5-02e2bdb517e8. [ 646.169451] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] [instance: 00e11309-0f82-49d5-b4f5-02e2bdb517e8] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 00e11309-0f82-49d5-b4f5-02e2bdb517e8. [ 646.169451] nova-conductor[53893]: DEBUG nova.network.neutron [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] [instance: 00e11309-0f82-49d5-b4f5-02e2bdb517e8] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 646.169451] nova-conductor[53893]: DEBUG nova.network.neutron [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] [instance: 00e11309-0f82-49d5-b4f5-02e2bdb517e8] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 646.169451] nova-conductor[53893]: DEBUG nova.network.neutron [None req-988f1a66-15d7-4e5a-ae4f-01a37abef9e8 tempest-DeleteServersAdminTestJSON-829365191 tempest-DeleteServersAdminTestJSON-829365191-project-member] [instance: 00e11309-0f82-49d5-b4f5-02e2bdb517e8] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 646.328500] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] [instance: 63bccd39-d951-4187-872d-559cd7fead30] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 855ab017-300e-4d43-a4cc-2423f56b7a11, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 63bccd39-d951-4187-872d-559cd7fead30 was re-scheduled: Binding failed for port 855ab017-300e-4d43-a4cc-2423f56b7a11, please check neutron logs for more information.\n'] [ 646.328500] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 646.328500] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 63bccd39-d951-4187-872d-559cd7fead30.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 63bccd39-d951-4187-872d-559cd7fead30. [ 646.328500] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] [instance: 63bccd39-d951-4187-872d-559cd7fead30] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 63bccd39-d951-4187-872d-559cd7fead30. [ 646.339472] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b79e4e0147144351b9ecbcfa32fc316e [ 646.340093] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Took 0.41 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 646.360017] nova-conductor[53894]: DEBUG nova.network.neutron [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] [instance: 63bccd39-d951-4187-872d-559cd7fead30] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 646.369746] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 646.369981] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 646.370197] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 646.399886] nova-conductor[53894]: DEBUG nova.network.neutron [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] [instance: 63bccd39-d951-4187-872d-559cd7fead30] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 646.404291] nova-conductor[53894]: DEBUG nova.network.neutron [None req-3a00aa79-4b70-4ea4-a6e7-a0e852900394 tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] [instance: 63bccd39-d951-4187-872d-559cd7fead30] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 646.426603] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 646.426849] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 646.427099] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 646.427425] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 646.427620] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 646.427796] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 646.444589] nova-conductor[53894]: DEBUG nova.quota [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Getting quotas for project fa911ec07ba14af1b645fa020802638c. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 646.449397] nova-conductor[53894]: DEBUG nova.quota [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Getting quotas for user 3999c3b2af1f4bc089587acda9cbd4e9 and project fa911ec07ba14af1b645fa020802638c. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 646.457345] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] [instance: 5aa53314-d177-4e8a-a2f5-ae0db9d30a66] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 646.457797] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 646.458064] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 646.458275] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 646.464217] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] [instance: 5aa53314-d177-4e8a-a2f5-ae0db9d30a66] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 646.465046] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 646.465301] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 646.465522] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 646.488151] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 646.488562] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 646.488562] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 647.265705] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Expecting reply to msg 61588b5053464918809a1880b53bfc4f in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 647.467045] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 61588b5053464918809a1880b53bfc4f [ 647.471020] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Took 0.21 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 647.496366] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 647.496800] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 647.497291] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 647.552254] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 647.552376] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 647.553419] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 647.553419] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 647.553419] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 647.553419] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 647.566521] nova-conductor[53894]: DEBUG nova.quota [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Getting quotas for project 751185c006ea414ea0ce2117380f13f9. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 647.570190] nova-conductor[53894]: DEBUG nova.quota [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Getting quotas for user 2b7538aa389a419f9928928eb07ee622 and project 751185c006ea414ea0ce2117380f13f9. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 647.578866] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] [instance: b908bdf3-a20f-4156-9ece-8dc038c9f749] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 647.578866] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 647.578866] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 647.578866] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 647.582995] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] [instance: b908bdf3-a20f-4156-9ece-8dc038c9f749] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 647.584134] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 647.584134] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 647.584134] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 647.606878] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 647.607523] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 647.607622] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 647.620146] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell0:SELECT=13 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 647.647952] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=51,nova_cell1:SAVEPOINT=4,nova_cell1:INSERT=100,nova_cell1:RELEASE=4 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 647.659356] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=55,nova_api:UPDATE=3,nova_api:DELETE=2 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 647.663995] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=65,nova_api:UPDATE=2,nova_api:DELETE=3 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 648.380382] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Expecting reply to msg 46d98f3cd71f4feb92413e219b539e50 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 648.574689] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 46d98f3cd71f4feb92413e219b539e50 [ 648.574689] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Took 0.20 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 648.591770] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 648.592174] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 648.592382] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 648.644986] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 648.646551] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 648.646551] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 648.646551] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 648.646551] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 648.646551] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 648.658964] nova-conductor[53894]: DEBUG nova.quota [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Getting quotas for project afc5e909ec5c4dd983ece5aa3236910f. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 648.664871] nova-conductor[53894]: DEBUG nova.quota [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Getting quotas for user ee5a21ff43314c1a857f6958056173f9 and project afc5e909ec5c4dd983ece5aa3236910f. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 648.672743] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: c0b7aef6-0633-420d-906b-818bcc54072e] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 648.672743] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 648.673499] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 648.673499] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 648.682280] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: c0b7aef6-0633-420d-906b-818bcc54072e] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 648.683016] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 648.683290] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 648.683384] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 648.716868] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 648.716868] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 648.716868] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 649.199080] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] [instance: 1385849e-9e64-4062-b6aa-300e6e7eab3d] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 17510c18-0039-426c-a5b8-1f817ff57d52, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 1385849e-9e64-4062-b6aa-300e6e7eab3d was re-scheduled: Binding failed for port 17510c18-0039-426c-a5b8-1f817ff57d52, please check neutron logs for more information.\n'] [ 649.199080] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 649.199080] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1385849e-9e64-4062-b6aa-300e6e7eab3d.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1385849e-9e64-4062-b6aa-300e6e7eab3d. [ 649.200447] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] [instance: 1385849e-9e64-4062-b6aa-300e6e7eab3d] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1385849e-9e64-4062-b6aa-300e6e7eab3d. [ 649.234704] nova-conductor[53894]: DEBUG nova.network.neutron [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] [instance: 1385849e-9e64-4062-b6aa-300e6e7eab3d] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 649.297614] nova-conductor[53894]: DEBUG nova.network.neutron [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] [instance: 1385849e-9e64-4062-b6aa-300e6e7eab3d] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 649.313646] nova-conductor[53894]: DEBUG nova.network.neutron [None req-05cc0ecb-8de9-477d-93f0-f68b8cf46bcc tempest-VolumesAssistedSnapshotsTest-541539088 tempest-VolumesAssistedSnapshotsTest-541539088-project-member] [instance: 1385849e-9e64-4062-b6aa-300e6e7eab3d] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 650.263242] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Expecting reply to msg f5cdf440622c40e49e3a6791fe00c20f in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 650.455648] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f5cdf440622c40e49e3a6791fe00c20f [ 650.455970] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Took 0.20 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 650.479129] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 650.479786] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 650.479786] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 650.528060] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 650.528060] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 650.528308] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 650.528458] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 650.528655] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 650.528815] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 650.542035] nova-conductor[53893]: DEBUG nova.quota [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Getting quotas for project 751185c006ea414ea0ce2117380f13f9. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 650.544988] nova-conductor[53893]: DEBUG nova.quota [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Getting quotas for user 2b7538aa389a419f9928928eb07ee622 and project 751185c006ea414ea0ce2117380f13f9. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 650.550316] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] [instance: a07bc185-7541-4a42-8c83-76cdcf157167] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 650.550863] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 650.551089] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 650.551263] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 650.559018] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] [instance: a07bc185-7541-4a42-8c83-76cdcf157167] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 650.559018] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 650.559018] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 650.559018] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 650.575146] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 650.575634] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 650.575948] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 655.790111] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Expecting reply to msg 51737a8455204a179efc01b68beb5560 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 655.963974] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 51737a8455204a179efc01b68beb5560 [ 655.964524] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Took 0.18 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 655.993910] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 655.994176] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 655.994441] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 656.036357] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 656.036638] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 656.036862] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 656.037324] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 656.037562] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 656.038373] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 656.052879] nova-conductor[53894]: DEBUG nova.quota [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Getting quotas for project b410a9b0a0a54c00bbbaa088cd81b957. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 656.055172] nova-conductor[53894]: DEBUG nova.quota [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Getting quotas for user a893b6f20a704d77968791ca48532894 and project b410a9b0a0a54c00bbbaa088cd81b957. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 656.063949] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] [instance: 03828801-4d8a-47dc-957e-f1aa64b652da] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 656.063949] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 656.064054] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 656.064154] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 656.076184] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] [instance: 03828801-4d8a-47dc-957e-f1aa64b652da] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 656.076546] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 656.076764] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 656.076937] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 656.103478] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 656.103478] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 656.103478] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 657.271813] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Expecting reply to msg 1c33baa3880548c5a12e7858a6deb405 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 657.491206] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1c33baa3880548c5a12e7858a6deb405 [ 657.491206] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Took 0.22 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 657.514683] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 657.515189] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 657.515189] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 657.624502] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 657.625858] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 657.625858] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 657.625858] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 657.625858] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 657.626383] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 657.642387] nova-conductor[53893]: DEBUG nova.quota [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Getting quotas for project c54606a0dde443f699e33acda0182e64. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 657.648395] nova-conductor[53893]: DEBUG nova.quota [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Getting quotas for user e5510866e61643bcb2d9c401fa9b9426 and project c54606a0dde443f699e33acda0182e64. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 657.656039] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] [instance: 467ba147-20b6-41eb-852f-4097cb45ba6b] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 657.657012] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 657.657012] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 657.657012] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 657.665502] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] [instance: 467ba147-20b6-41eb-852f-4097cb45ba6b] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 657.666327] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 657.666561] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 657.666741] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 657.714958] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 657.714958] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 657.714958] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 661.980770] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Expecting reply to msg 2693fbb8c1b64c939c9ddefebc5de875 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 662.098603] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] [instance: 5aa53314-d177-4e8a-a2f5-ae0db9d30a66] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 1ecca70e-b1c0-4563-8b14-d6cdc6b92ebe, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 5aa53314-d177-4e8a-a2f5-ae0db9d30a66 was re-scheduled: Binding failed for port 1ecca70e-b1c0-4563-8b14-d6cdc6b92ebe, please check neutron logs for more information.\n'] [ 662.099755] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 662.100359] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 5aa53314-d177-4e8a-a2f5-ae0db9d30a66.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 5aa53314-d177-4e8a-a2f5-ae0db9d30a66. [ 662.101700] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] [instance: 5aa53314-d177-4e8a-a2f5-ae0db9d30a66] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 5aa53314-d177-4e8a-a2f5-ae0db9d30a66. [ 662.140681] nova-conductor[53893]: DEBUG nova.network.neutron [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] [instance: 5aa53314-d177-4e8a-a2f5-ae0db9d30a66] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 662.224213] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2693fbb8c1b64c939c9ddefebc5de875 [ 662.225031] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Took 0.25 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 662.246868] nova-conductor[53893]: DEBUG nova.network.neutron [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] [instance: 5aa53314-d177-4e8a-a2f5-ae0db9d30a66] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 662.252209] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 662.252497] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 662.252961] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 662.258881] nova-conductor[53893]: DEBUG nova.network.neutron [None req-8b6d1d73-2883-43ce-afed-5f004f6093e2 tempest-ServerActionsTestJSON-1372202196 tempest-ServerActionsTestJSON-1372202196-project-member] [instance: 5aa53314-d177-4e8a-a2f5-ae0db9d30a66] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 662.343725] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 662.343725] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 662.344290] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 662.345192] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 662.345192] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 662.345192] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 662.366432] nova-conductor[53894]: DEBUG nova.quota [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Getting quotas for project ada332ead1f5480191a0bd8bf9496847. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 662.371077] nova-conductor[53894]: DEBUG nova.quota [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Getting quotas for user 32cf35f7cdf54e61ba76eca15c5b6045 and project ada332ead1f5480191a0bd8bf9496847. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 662.378477] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] [instance: 88c75c70-0ed0-4f19-bdb5-60bb07307b26] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 662.379303] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 662.379623] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 662.379867] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 662.388408] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] [instance: 88c75c70-0ed0-4f19-bdb5-60bb07307b26] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 662.388908] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 662.389273] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 662.389469] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 662.418286] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 662.418524] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 662.418702] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 663.035640] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: c0b7aef6-0633-420d-906b-818bcc54072e] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 07c16f5c-e943-48a0-81a3-e7f9ee9ca363, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance c0b7aef6-0633-420d-906b-818bcc54072e was re-scheduled: Binding failed for port 07c16f5c-e943-48a0-81a3-e7f9ee9ca363, please check neutron logs for more information.\n'] [ 663.036265] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 663.036645] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance c0b7aef6-0633-420d-906b-818bcc54072e.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance c0b7aef6-0633-420d-906b-818bcc54072e. [ 663.036744] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: c0b7aef6-0633-420d-906b-818bcc54072e] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance c0b7aef6-0633-420d-906b-818bcc54072e. [ 663.064855] nova-conductor[53893]: DEBUG nova.network.neutron [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: c0b7aef6-0633-420d-906b-818bcc54072e] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 663.205598] nova-conductor[53893]: DEBUG nova.network.neutron [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: c0b7aef6-0633-420d-906b-818bcc54072e] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 663.208183] nova-conductor[53893]: DEBUG nova.network.neutron [None req-60b7bf0a-a10e-4ad3-8b7f-31d0ceba0e55 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: c0b7aef6-0633-420d-906b-818bcc54072e] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 663.755017] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] [instance: b908bdf3-a20f-4156-9ece-8dc038c9f749] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 2caa9152-e1cc-4d04-9e73-221b56b1f580, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance b908bdf3-a20f-4156-9ece-8dc038c9f749 was re-scheduled: Binding failed for port 2caa9152-e1cc-4d04-9e73-221b56b1f580, please check neutron logs for more information.\n'] [ 663.755017] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 663.755404] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b908bdf3-a20f-4156-9ece-8dc038c9f749.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b908bdf3-a20f-4156-9ece-8dc038c9f749. [ 663.755457] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] [instance: b908bdf3-a20f-4156-9ece-8dc038c9f749] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b908bdf3-a20f-4156-9ece-8dc038c9f749. [ 663.784595] nova-conductor[53894]: DEBUG nova.network.neutron [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] [instance: b908bdf3-a20f-4156-9ece-8dc038c9f749] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 663.861692] nova-conductor[53894]: DEBUG nova.network.neutron [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] [instance: b908bdf3-a20f-4156-9ece-8dc038c9f749] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 663.868573] nova-conductor[53894]: DEBUG nova.network.neutron [None req-5836cce4-2eb7-4522-95a9-9d5e8a10c623 tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] [instance: b908bdf3-a20f-4156-9ece-8dc038c9f749] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 664.487805] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Expecting reply to msg 46665400a6e6420ca3bcee8c592fa5f2 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 664.658770] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 46665400a6e6420ca3bcee8c592fa5f2 [ 664.658770] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Took 0.18 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 664.673652] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 664.673991] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 664.674134] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 664.714768] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 664.715046] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 664.715238] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 664.715918] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 664.716151] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 664.716320] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 664.736129] nova-conductor[53894]: DEBUG nova.quota [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Getting quotas for project 387cec2bf6194b85a24dec9167f9e2a3. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 664.742108] nova-conductor[53894]: DEBUG nova.quota [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Getting quotas for user ac9d079b0e3244f6bf0871a4eea21b49 and project 387cec2bf6194b85a24dec9167f9e2a3. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 664.752782] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] [instance: e9f942ac-414f-4d15-9b7a-c26e6612a787] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 664.753250] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 664.753669] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 664.754461] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 664.766459] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] [instance: e9f942ac-414f-4d15-9b7a-c26e6612a787] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 664.768957] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 664.768957] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 664.768957] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 664.796209] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 664.797440] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 664.797440] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 664.931017] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=707,nova_cell1:UPDATE=203,nova_cell1:INSERT=29,nova_cell1:SAVEPOINT=30,nova_cell1:RELEASE=30 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 667.634017] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell0:SELECT=4 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 667.672294] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=18,nova_cell1:INSERT=39,nova_cell1:SAVEPOINT=1,nova_cell1:RELEASE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 667.708708] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=24,nova_api:UPDATE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 667.708708] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=24,nova_api:UPDATE=1,nova_api:DELETE=2 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 667.713404] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] [instance: 04fd7039-c2c8-4b78-8c3d-37eb66fe2115] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn\n self._fetch_image_if_missing(context, vi)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing\n image_cache(vi, tmp_image_ds_loc)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image\n vm_util.copy_virtual_disk(\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk\n session._wait_for_task(vmdk_copy_task)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task\n return self.wait_for_task(task_ref)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task\n return evt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner\n self.f(*self.args, **self.kw)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task\n raise exceptions.translate_fault(task_info.error)\n', "oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n", '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', "nova.exception.RescheduledException: Build of instance 04fd7039-c2c8-4b78-8c3d-37eb66fe2115 was re-scheduled: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n"] [ 667.715328] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 667.715862] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 04fd7039-c2c8-4b78-8c3d-37eb66fe2115.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 04fd7039-c2c8-4b78-8c3d-37eb66fe2115. [ 667.716430] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-d5b2c684-c0d4-41f5-b9a1-c187a4e2c597 tempest-ServersAdmin275Test-1948904895 tempest-ServersAdmin275Test-1948904895-project-member] [instance: 04fd7039-c2c8-4b78-8c3d-37eb66fe2115] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 04fd7039-c2c8-4b78-8c3d-37eb66fe2115. [ 668.573451] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] [instance: a07bc185-7541-4a42-8c83-76cdcf157167] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port ee5f9094-c2bd-4d83-b04f-0fb72ece6737, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance a07bc185-7541-4a42-8c83-76cdcf157167 was re-scheduled: Binding failed for port ee5f9094-c2bd-4d83-b04f-0fb72ece6737, please check neutron logs for more information.\n'] [ 668.575152] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 668.575441] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance a07bc185-7541-4a42-8c83-76cdcf157167.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance a07bc185-7541-4a42-8c83-76cdcf157167. [ 668.575662] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] [instance: a07bc185-7541-4a42-8c83-76cdcf157167] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance a07bc185-7541-4a42-8c83-76cdcf157167. [ 668.609964] nova-conductor[53893]: DEBUG nova.network.neutron [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] [instance: a07bc185-7541-4a42-8c83-76cdcf157167] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 668.703133] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Expecting reply to msg e6a381d81feb4bd2b6edaaea515a0b48 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 668.705771] nova-conductor[53893]: DEBUG nova.network.neutron [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] [instance: a07bc185-7541-4a42-8c83-76cdcf157167] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 668.713060] nova-conductor[53893]: DEBUG nova.network.neutron [None req-cf869085-50f6-4df8-9fc1-eae82e39c36d tempest-ServersAdminTestJSON-1599288174 tempest-ServersAdminTestJSON-1599288174-project-member] [instance: a07bc185-7541-4a42-8c83-76cdcf157167] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 668.872709] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e6a381d81feb4bd2b6edaaea515a0b48 [ 668.872957] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Took 0.17 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 668.899321] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 668.899321] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 668.899321] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 668.972502] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 668.972950] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 668.975033] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 668.975033] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 668.975033] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 668.975033] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 668.988865] nova-conductor[53893]: DEBUG nova.quota [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Getting quotas for project d3952a0eb9b246b3981a76df98b855f0. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 668.992202] nova-conductor[53893]: DEBUG nova.quota [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Getting quotas for user 39dd18ce542f42e08215016404ffe9dc and project d3952a0eb9b246b3981a76df98b855f0. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 668.999764] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] [instance: 410d3c8b-9be0-4863-b121-c9acffae69e4] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 669.002504] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 669.002504] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 669.002504] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 669.005945] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] [instance: 410d3c8b-9be0-4863-b121-c9acffae69e4] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 669.006946] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 669.009630] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 669.009630] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 669.034338] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 669.034338] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 669.034579] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 670.423823] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Expecting reply to msg 004b53b8df9f4f05a37d50b3fff85074 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 670.665460] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 004b53b8df9f4f05a37d50b3fff85074 [ 670.666308] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Took 0.25 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 670.686128] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 670.686402] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 670.686582] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 670.741306] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=828,nova_cell1:UPDATE=230,nova_cell1:INSERT=37,nova_cell1:SAVEPOINT=35,nova_cell1:RELEASE=35 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 670.743699] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 670.744225] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 670.744545] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 670.745847] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 670.746236] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 670.748363] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 670.760268] nova-conductor[53894]: DEBUG nova.quota [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Getting quotas for project ab80091b23ff48bf803d575c6a8a55f9. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 670.765939] nova-conductor[53894]: DEBUG nova.quota [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Getting quotas for user c4ea738bdd9b4b08ad5bcd2b59402c13 and project ab80091b23ff48bf803d575c6a8a55f9. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 670.770346] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] [instance: 996e9c89-dd9c-4c76-952d-9ff90dec38c1] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 670.770838] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 670.771295] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 670.771295] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 670.783270] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] [instance: 996e9c89-dd9c-4c76-952d-9ff90dec38c1] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 670.783270] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 670.783270] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 670.783270] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 670.807931] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 670.808636] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 670.808636] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 674.253377] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] [instance: 03828801-4d8a-47dc-957e-f1aa64b652da] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port ec5f7e6a-c76a-4424-a98d-f10c2bdf13cc, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 03828801-4d8a-47dc-957e-f1aa64b652da was re-scheduled: Binding failed for port ec5f7e6a-c76a-4424-a98d-f10c2bdf13cc, please check neutron logs for more information.\n'] [ 674.254245] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 674.254584] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 03828801-4d8a-47dc-957e-f1aa64b652da.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 03828801-4d8a-47dc-957e-f1aa64b652da. [ 674.255274] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] [instance: 03828801-4d8a-47dc-957e-f1aa64b652da] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 03828801-4d8a-47dc-957e-f1aa64b652da. [ 674.297375] nova-conductor[53894]: DEBUG nova.network.neutron [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] [instance: 03828801-4d8a-47dc-957e-f1aa64b652da] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 674.406316] nova-conductor[53894]: DEBUG nova.network.neutron [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] [instance: 03828801-4d8a-47dc-957e-f1aa64b652da] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 674.415434] nova-conductor[53894]: DEBUG nova.network.neutron [None req-a9e6f812-044e-444b-951c-26c125076904 tempest-VolumesAdminNegativeTest-225605931 tempest-VolumesAdminNegativeTest-225605931-project-member] [instance: 03828801-4d8a-47dc-957e-f1aa64b652da] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 675.450316] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_api:SELECT=151,nova_api:UPDATE=4,nova_api:DELETE=6 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 675.452388] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_api:SELECT=162,nova_api:UPDATE=9,nova_api:DELETE=7 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 675.459013] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Expecting reply to msg 96751b647b004c8dbc3c8828b636cc3d in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 675.620255] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 96751b647b004c8dbc3c8828b636cc3d [ 675.620255] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Took 0.16 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 675.636663] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 675.637243] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 675.637243] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 675.639631] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=121,nova_cell1:SAVEPOINT=10,nova_cell1:INSERT=257,nova_cell1:RELEASE=10 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 675.692655] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 675.692759] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 675.692974] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 675.693395] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 675.693632] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 675.693873] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 675.703729] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell0:SELECT=30 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 675.726145] nova-conductor[53894]: DEBUG nova.quota [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Getting quotas for project afc5e909ec5c4dd983ece5aa3236910f. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 675.729182] nova-conductor[53894]: DEBUG nova.quota [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Getting quotas for user ee5a21ff43314c1a857f6958056173f9 and project afc5e909ec5c4dd983ece5aa3236910f. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 675.737788] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: f2b1637b-6a4b-43e7-bd5d-d6f33abd9867] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 675.738346] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 675.738648] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 675.738837] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 675.743798] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: f2b1637b-6a4b-43e7-bd5d-d6f33abd9867] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 675.744540] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 675.745054] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 675.745163] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 675.771487] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 675.771487] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 675.771487] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 675.827087] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] [instance: 467ba147-20b6-41eb-852f-4097cb45ba6b] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 48426ae3-8620-4cde-b703-aa3f7b25ab22, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 467ba147-20b6-41eb-852f-4097cb45ba6b was re-scheduled: Binding failed for port 48426ae3-8620-4cde-b703-aa3f7b25ab22, please check neutron logs for more information.\n'] [ 675.827925] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 675.828225] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 467ba147-20b6-41eb-852f-4097cb45ba6b.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 467ba147-20b6-41eb-852f-4097cb45ba6b. [ 675.828630] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] [instance: 467ba147-20b6-41eb-852f-4097cb45ba6b] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 467ba147-20b6-41eb-852f-4097cb45ba6b. [ 675.865111] nova-conductor[53894]: DEBUG nova.network.neutron [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] [instance: 467ba147-20b6-41eb-852f-4097cb45ba6b] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 676.222335] nova-conductor[53894]: DEBUG nova.network.neutron [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] [instance: 467ba147-20b6-41eb-852f-4097cb45ba6b] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 676.227578] nova-conductor[53894]: DEBUG nova.network.neutron [None req-21eeb9c5-8c2e-4625-9e3f-771252dc284c tempest-ServersWithSpecificFlavorTestJSON-2109930540 tempest-ServersWithSpecificFlavorTestJSON-2109930540-project-member] [instance: 467ba147-20b6-41eb-852f-4097cb45ba6b] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 676.823134] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] [instance: 88c75c70-0ed0-4f19-bdb5-60bb07307b26] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port d5c3fc88-4154-47af-858d-9db76308a133, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 88c75c70-0ed0-4f19-bdb5-60bb07307b26 was re-scheduled: Binding failed for port d5c3fc88-4154-47af-858d-9db76308a133, please check neutron logs for more information.\n'] [ 676.823134] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 676.823454] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 88c75c70-0ed0-4f19-bdb5-60bb07307b26.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 88c75c70-0ed0-4f19-bdb5-60bb07307b26. [ 676.824245] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] [instance: 88c75c70-0ed0-4f19-bdb5-60bb07307b26] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 88c75c70-0ed0-4f19-bdb5-60bb07307b26. [ 676.848538] nova-conductor[53893]: DEBUG nova.network.neutron [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] [instance: 88c75c70-0ed0-4f19-bdb5-60bb07307b26] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 676.939791] nova-conductor[53893]: DEBUG nova.network.neutron [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] [instance: 88c75c70-0ed0-4f19-bdb5-60bb07307b26] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 676.945889] nova-conductor[53893]: DEBUG nova.network.neutron [None req-2c8c43ec-9620-40ac-be02-52db2aedd1d3 tempest-FloatingIPsAssociationTestJSON-1985048995 tempest-FloatingIPsAssociationTestJSON-1985048995-project-member] [instance: 88c75c70-0ed0-4f19-bdb5-60bb07307b26] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 678.982040] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell0:SELECT=2 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 679.011499] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=9,nova_cell1:INSERT=19 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 679.031034] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=11 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 679.033502] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 682.286235] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] [instance: e9f942ac-414f-4d15-9b7a-c26e6612a787] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port dc6c2510-4dcc-4d49-ad6f-a8b612cfa398, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance e9f942ac-414f-4d15-9b7a-c26e6612a787 was re-scheduled: Binding failed for port dc6c2510-4dcc-4d49-ad6f-a8b612cfa398, please check neutron logs for more information.\n'] [ 682.287418] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 682.287680] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e9f942ac-414f-4d15-9b7a-c26e6612a787.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e9f942ac-414f-4d15-9b7a-c26e6612a787. [ 682.288017] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] [instance: e9f942ac-414f-4d15-9b7a-c26e6612a787] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e9f942ac-414f-4d15-9b7a-c26e6612a787. [ 682.316317] nova-conductor[53894]: DEBUG nova.network.neutron [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] [instance: e9f942ac-414f-4d15-9b7a-c26e6612a787] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 682.383815] nova-conductor[53894]: DEBUG nova.network.neutron [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] [instance: e9f942ac-414f-4d15-9b7a-c26e6612a787] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 682.389957] nova-conductor[53894]: DEBUG nova.network.neutron [None req-8aced41d-2fae-4c5f-a1ea-e31e722dd354 tempest-ImagesOneServerNegativeTestJSON-1423847491 tempest-ImagesOneServerNegativeTestJSON-1423847491-project-member] [instance: e9f942ac-414f-4d15-9b7a-c26e6612a787] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 682.616938] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Expecting reply to msg 84bccb258e584c63a8a67737725c4eab in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 682.784196] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 84bccb258e584c63a8a67737725c4eab [ 682.785019] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Took 0.17 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 682.801016] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 682.801302] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 682.801516] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 682.862305] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 682.862507] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 682.862686] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 682.863069] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 682.863259] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 682.863445] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 682.873345] nova-conductor[53894]: DEBUG nova.quota [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Getting quotas for project 56f718ba688d493793b74486ea1f9474. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 682.875642] nova-conductor[53894]: DEBUG nova.quota [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Getting quotas for user 448e504d26b447f5bbd99acd33ada1a8 and project 56f718ba688d493793b74486ea1f9474. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 682.887238] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] [instance: 4858bdbd-3aa6-4ac6-ab02-cd2d6ff03ea2] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 682.887690] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 682.887897] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 682.888076] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 682.894519] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] [instance: 4858bdbd-3aa6-4ac6-ab02-cd2d6ff03ea2] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 682.895191] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 682.895412] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 682.895586] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 682.913470] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 682.913692] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 682.913865] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 685.527127] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Expecting reply to msg 7c4b1e43528e45998cdf63b22d0c2359 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 685.726379] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7c4b1e43528e45998cdf63b22d0c2359 [ 685.727226] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Took 0.20 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 685.748058] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 685.748346] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 685.748523] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 685.814014] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 685.814827] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 685.814827] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 685.815054] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 685.815865] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 685.815865] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 685.827927] nova-conductor[53894]: DEBUG nova.quota [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Getting quotas for project 84e748c48ab44d33b8b34330bd643cbb. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 685.831276] nova-conductor[53894]: DEBUG nova.quota [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Getting quotas for user 51bd8a6208de4c3984a79dc5ba3ba622 and project 84e748c48ab44d33b8b34330bd643cbb. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 685.837920] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] [instance: e42594f1-7bf7-4630-9ebf-950007812a14] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 685.838765] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 685.838974] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 685.839209] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 685.846023] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] [instance: e42594f1-7bf7-4630-9ebf-950007812a14] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 685.846762] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 685.847089] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 685.847285] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 685.896726] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 685.896918] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 685.897123] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 687.574719] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] [instance: 996e9c89-dd9c-4c76-952d-9ff90dec38c1] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 6fc0ede2-587a-4953-acb8-f92ad2c564f4, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 996e9c89-dd9c-4c76-952d-9ff90dec38c1 was re-scheduled: Binding failed for port 6fc0ede2-587a-4953-acb8-f92ad2c564f4, please check neutron logs for more information.\n'] [ 687.575293] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 687.575524] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 996e9c89-dd9c-4c76-952d-9ff90dec38c1.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 996e9c89-dd9c-4c76-952d-9ff90dec38c1. [ 687.576152] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] [instance: 996e9c89-dd9c-4c76-952d-9ff90dec38c1] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 996e9c89-dd9c-4c76-952d-9ff90dec38c1. [ 687.599099] nova-conductor[53893]: DEBUG nova.network.neutron [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] [instance: 996e9c89-dd9c-4c76-952d-9ff90dec38c1] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 687.691273] nova-conductor[53893]: DEBUG nova.network.neutron [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] [instance: 996e9c89-dd9c-4c76-952d-9ff90dec38c1] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 687.696178] nova-conductor[53893]: DEBUG nova.network.neutron [None req-62cc2796-e288-4c3c-b946-37259e71a2d6 tempest-ServerDiagnosticsNegativeTest-1789156982 tempest-ServerDiagnosticsNegativeTest-1789156982-project-member] [instance: 996e9c89-dd9c-4c76-952d-9ff90dec38c1] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 689.325079] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] [instance: 410d3c8b-9be0-4863-b121-c9acffae69e4] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 087fd1ae-48bb-4dec-b57e-45601c385600, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 410d3c8b-9be0-4863-b121-c9acffae69e4 was re-scheduled: Binding failed for port 087fd1ae-48bb-4dec-b57e-45601c385600, please check neutron logs for more information.\n'] [ 689.326217] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 689.326531] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 410d3c8b-9be0-4863-b121-c9acffae69e4.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 410d3c8b-9be0-4863-b121-c9acffae69e4. [ 689.326663] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] [instance: 410d3c8b-9be0-4863-b121-c9acffae69e4] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 410d3c8b-9be0-4863-b121-c9acffae69e4. [ 689.369209] nova-conductor[53893]: DEBUG nova.network.neutron [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] [instance: 410d3c8b-9be0-4863-b121-c9acffae69e4] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 689.465770] nova-conductor[53893]: DEBUG nova.network.neutron [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] [instance: 410d3c8b-9be0-4863-b121-c9acffae69e4] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 689.468968] nova-conductor[53893]: DEBUG nova.network.neutron [None req-5a9b7fcd-0f28-4899-91fb-ff7b93a56f7d tempest-ServersTestMultiNic-2009464467 tempest-ServersTestMultiNic-2009464467-project-member] [instance: 410d3c8b-9be0-4863-b121-c9acffae69e4] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 689.711196] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Expecting reply to msg 3bcc865a86634d948fbbd04caa503452 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 689.930266] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 3bcc865a86634d948fbbd04caa503452 [ 689.931478] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Took 0.22 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 689.951199] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 689.951439] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 689.951615] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 690.009103] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 690.009348] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 690.009519] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 690.009969] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 690.010171] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 690.010330] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 690.022613] nova-conductor[53894]: DEBUG nova.quota [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Getting quotas for project 3dfb77f12805418eaa6127fc75becec0. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 690.025376] nova-conductor[53894]: DEBUG nova.quota [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Getting quotas for user 789177a2f7be455cadec45cf03d67521 and project 3dfb77f12805418eaa6127fc75becec0. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 690.033629] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: ec1dcbbc-16df-4965-a61e-6c70e4c6c0ba] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 690.034154] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 690.034964] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 690.035542] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 690.042076] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: ec1dcbbc-16df-4965-a61e-6c70e4c6c0ba] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 690.042076] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 690.042076] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 690.042076] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 690.061990] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 690.062220] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 690.062988] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 690.808286] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Expecting reply to msg cbac1255fd8a494c9cbbef8d220926a8 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 690.861995] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Expecting reply to msg 7b3e92ce7e7941a5809f26472c9c07f5 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 690.981858] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg cbac1255fd8a494c9cbbef8d220926a8 [ 690.983726] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Took 0.18 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 691.001886] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 691.001983] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 691.002570] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 691.009523] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7b3e92ce7e7941a5809f26472c9c07f5 [ 691.010213] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Took 0.15 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 691.027960] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 691.027960] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 691.027960] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 691.068633] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 691.068861] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 691.069054] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 691.069432] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 691.069617] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 691.069807] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 691.106012] nova-conductor[53894]: DEBUG nova.quota [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Getting quotas for project fc3d6b59e1fc476f83e4d63e858ca18c. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 691.110437] nova-conductor[53894]: DEBUG nova.quota [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Getting quotas for user 4ee6f0e9339c48fa984086efe0bfde44 and project fc3d6b59e1fc476f83e4d63e858ca18c. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 691.112575] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 691.112575] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 691.112575] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 691.112806] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 691.112994] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 691.113486] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 691.130110] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] [instance: bfa7d6f2-050a-4604-8f9c-250d89b091a5] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 691.131016] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 691.131016] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 691.131016] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 691.134722] nova-conductor[53894]: DEBUG nova.quota [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Getting quotas for project 445252cbff1b49cf8874f3ae2e2e3395. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 691.137260] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] [instance: bfa7d6f2-050a-4604-8f9c-250d89b091a5] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 691.137888] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 691.138106] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 691.138362] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 691.139359] nova-conductor[53894]: DEBUG nova.quota [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Getting quotas for user 2048b07a11f84e9a8ccebf543f504aab and project 445252cbff1b49cf8874f3ae2e2e3395. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 691.148925] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] [instance: fc403a94-8ac7-4f73-a0a8-51539dcc47d4] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 691.149384] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 691.149594] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 691.149818] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 691.158642] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] [instance: fc403a94-8ac7-4f73-a0a8-51539dcc47d4] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 691.159351] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 691.159565] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 691.159775] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 691.184346] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 691.184574] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 691.184756] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 691.191171] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 691.191403] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 691.191595] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 692.576417] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: f2b1637b-6a4b-43e7-bd5d-d6f33abd9867] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port a0a91f44-3ef8-4505-ba8d-6a2e00e6853d, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance f2b1637b-6a4b-43e7-bd5d-d6f33abd9867 was re-scheduled: Binding failed for port a0a91f44-3ef8-4505-ba8d-6a2e00e6853d, please check neutron logs for more information.\n'] [ 692.576417] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 692.576834] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance f2b1637b-6a4b-43e7-bd5d-d6f33abd9867.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance f2b1637b-6a4b-43e7-bd5d-d6f33abd9867. [ 692.578598] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: f2b1637b-6a4b-43e7-bd5d-d6f33abd9867] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance f2b1637b-6a4b-43e7-bd5d-d6f33abd9867. [ 692.601557] nova-conductor[53893]: DEBUG nova.network.neutron [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: f2b1637b-6a4b-43e7-bd5d-d6f33abd9867] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 692.697060] nova-conductor[53893]: DEBUG nova.network.neutron [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: f2b1637b-6a4b-43e7-bd5d-d6f33abd9867] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 692.700602] nova-conductor[53893]: DEBUG nova.network.neutron [None req-4659b6a2-01b5-4b63-89f9-ed4c983c70a3 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: f2b1637b-6a4b-43e7-bd5d-d6f33abd9867] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 695.666313] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Expecting reply to msg e4022d4cc6d8438b9a09980036fabb2b in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 695.809880] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Expecting reply to msg d0d0a39a8826415dbf514b3c7ee31171 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 695.835435] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e4022d4cc6d8438b9a09980036fabb2b [ 695.836329] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Took 0.17 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 695.859951] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 695.861177] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 695.861448] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 695.932296] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 695.932631] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 695.932833] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 695.933431] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 695.933616] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 695.933778] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 695.956244] nova-conductor[53894]: DEBUG nova.quota [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Getting quotas for project 1a71ad350ba241b5a3f933887cb0e8a4. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 695.962182] nova-conductor[53894]: DEBUG nova.quota [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Getting quotas for user 084fb304b8aa400ebcd3bab4404c36ae and project 1a71ad350ba241b5a3f933887cb0e8a4. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 695.975765] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] [instance: deb53e97-7f20-47d3-a069-0e435776bad8] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 695.976321] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 695.976525] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 695.976699] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 695.984098] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] [instance: deb53e97-7f20-47d3-a069-0e435776bad8] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 695.984821] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 695.985034] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 695.985208] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 696.002625] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d0d0a39a8826415dbf514b3c7ee31171 [ 696.003456] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Took 0.20 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 696.028072] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 696.028315] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 696.028488] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 696.033266] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 696.033519] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 696.033690] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 696.126981] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 696.127429] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 696.127885] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 696.128400] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 696.129352] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 696.129697] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 696.155769] nova-conductor[53893]: DEBUG nova.quota [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Getting quotas for project 19b2170b4bde4207b5a774f1cd266796. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 696.168408] nova-conductor[53893]: DEBUG nova.quota [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Getting quotas for user 38ed2b39a2d0462095de83291bca4587 and project 19b2170b4bde4207b5a774f1cd266796. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 696.195439] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] [instance: e20b5017-bc1a-41c0-ba4b-83e4df43a53d] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 696.195961] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 696.196186] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 696.196355] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 696.208113] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] [instance: e20b5017-bc1a-41c0-ba4b-83e4df43a53d] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 696.209081] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 696.209339] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 696.209486] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 696.242823] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 696.243150] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 696.243475] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 698.841640] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] [instance: 4858bdbd-3aa6-4ac6-ab02-cd2d6ff03ea2] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 9ae290d0-490e-4e1f-b088-547ba11d2802, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 4858bdbd-3aa6-4ac6-ab02-cd2d6ff03ea2 was re-scheduled: Binding failed for port 9ae290d0-490e-4e1f-b088-547ba11d2802, please check neutron logs for more information.\n'] [ 698.842712] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 698.843252] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 4858bdbd-3aa6-4ac6-ab02-cd2d6ff03ea2.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 4858bdbd-3aa6-4ac6-ab02-cd2d6ff03ea2. [ 698.843623] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] [instance: 4858bdbd-3aa6-4ac6-ab02-cd2d6ff03ea2] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 4858bdbd-3aa6-4ac6-ab02-cd2d6ff03ea2. [ 698.873044] nova-conductor[53893]: DEBUG nova.network.neutron [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] [instance: 4858bdbd-3aa6-4ac6-ab02-cd2d6ff03ea2] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 698.954755] nova-conductor[53893]: DEBUG nova.network.neutron [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] [instance: 4858bdbd-3aa6-4ac6-ab02-cd2d6ff03ea2] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 698.963130] nova-conductor[53893]: DEBUG nova.network.neutron [None req-3fe887b7-08ae-4246-9828-305b72996d5b tempest-ServerActionsTestOtherB-276147654 tempest-ServerActionsTestOtherB-276147654-project-member] [instance: 4858bdbd-3aa6-4ac6-ab02-cd2d6ff03ea2] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 702.934542] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Expecting reply to msg c9443da992af4746bcf3541af65654e1 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 703.023107] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Expecting reply to msg 90380b68d5564d7a97b66165a5e4bb68 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 703.105268] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c9443da992af4746bcf3541af65654e1 [ 703.105268] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Took 0.17 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 703.124928] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 703.125234] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 703.125915] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 703.177562] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 703.177728] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 703.177979] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 703.178466] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 90380b68d5564d7a97b66165a5e4bb68 [ 703.179556] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Took 0.16 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 703.181064] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 703.181707] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.003s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 703.181707] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 703.191219] nova-conductor[53893]: DEBUG nova.quota [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Getting quotas for project f01800739b6848ce82622704b8ab4b2a. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 703.194604] nova-conductor[53893]: DEBUG nova.quota [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Getting quotas for user 35d0551ce6c241a2b5db99a60c6ce94b and project f01800739b6848ce82622704b8ab4b2a. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 703.204630] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] [instance: b3e22378-e257-4e49-9eb2-787b4afd0eb8] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 703.204630] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 703.204630] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 703.204630] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 703.208408] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] [instance: b3e22378-e257-4e49-9eb2-787b4afd0eb8] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 703.209353] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 703.209353] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 703.209353] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 703.211518] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 703.211789] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 703.211988] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 703.234596] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 703.234862] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 703.235110] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 703.263235] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 703.263235] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 703.263235] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 703.263235] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 703.263610] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 703.263610] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 703.290904] nova-conductor[53894]: DEBUG nova.quota [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Getting quotas for project b1a43347e40844d4a9c866aeabeb5470. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 703.299168] nova-conductor[53894]: DEBUG nova.quota [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Getting quotas for user 93fabaa5ad29483d870b131eac8d2677 and project b1a43347e40844d4a9c866aeabeb5470. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 703.325373] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] [instance: 7b2debf8-278d-443f-aaf7-3ae6c129981c] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 703.325977] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 703.326155] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 703.326341] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 703.337052] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] [instance: 7b2debf8-278d-443f-aaf7-3ae6c129981c] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 703.337804] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 703.338043] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 703.338220] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 703.373377] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 703.373377] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 703.373377] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 704.968640] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: ec1dcbbc-16df-4965-a61e-6c70e4c6c0ba] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port b05ad338-4b4c-4909-b4c5-4725bd07c46b, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance ec1dcbbc-16df-4965-a61e-6c70e4c6c0ba was re-scheduled: Binding failed for port b05ad338-4b4c-4909-b4c5-4725bd07c46b, please check neutron logs for more information.\n'] [ 704.969318] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 704.969676] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ec1dcbbc-16df-4965-a61e-6c70e4c6c0ba.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ec1dcbbc-16df-4965-a61e-6c70e4c6c0ba. [ 704.970085] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: ec1dcbbc-16df-4965-a61e-6c70e4c6c0ba] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ec1dcbbc-16df-4965-a61e-6c70e4c6c0ba. [ 705.005667] nova-conductor[53893]: DEBUG nova.network.neutron [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: ec1dcbbc-16df-4965-a61e-6c70e4c6c0ba] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 705.117222] nova-conductor[53893]: DEBUG nova.network.neutron [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: ec1dcbbc-16df-4965-a61e-6c70e4c6c0ba] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 705.120617] nova-conductor[53893]: DEBUG nova.network.neutron [None req-1ee23a03-1461-4cd6-ac45-deb0dcdb6386 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: ec1dcbbc-16df-4965-a61e-6c70e4c6c0ba] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 705.672284] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] [instance: e42594f1-7bf7-4630-9ebf-950007812a14] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 24b937ea-bba2-4255-a685-87739ea2b89e, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance e42594f1-7bf7-4630-9ebf-950007812a14 was re-scheduled: Binding failed for port 24b937ea-bba2-4255-a685-87739ea2b89e, please check neutron logs for more information.\n'] [ 705.673247] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 705.673532] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e42594f1-7bf7-4630-9ebf-950007812a14.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e42594f1-7bf7-4630-9ebf-950007812a14. [ 705.673997] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] [instance: e42594f1-7bf7-4630-9ebf-950007812a14] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e42594f1-7bf7-4630-9ebf-950007812a14. [ 705.705848] nova-conductor[53894]: DEBUG nova.network.neutron [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] [instance: e42594f1-7bf7-4630-9ebf-950007812a14] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 705.776357] nova-conductor[53894]: DEBUG nova.network.neutron [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] [instance: e42594f1-7bf7-4630-9ebf-950007812a14] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 705.781794] nova-conductor[53894]: DEBUG nova.network.neutron [None req-c58d93f3-90fc-409d-9c3a-8bc529b434dd tempest-ServerAddressesTestJSON-475267544 tempest-ServerAddressesTestJSON-475267544-project-member] [instance: e42594f1-7bf7-4630-9ebf-950007812a14] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 705.962927] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] [instance: fc403a94-8ac7-4f73-a0a8-51539dcc47d4] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 80d114f8-4d5f-41a9-b5ab-bb2d668a8121, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance fc403a94-8ac7-4f73-a0a8-51539dcc47d4 was re-scheduled: Binding failed for port 80d114f8-4d5f-41a9-b5ab-bb2d668a8121, please check neutron logs for more information.\n'] [ 705.963485] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 705.963720] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance fc403a94-8ac7-4f73-a0a8-51539dcc47d4.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance fc403a94-8ac7-4f73-a0a8-51539dcc47d4. [ 705.963930] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] [instance: fc403a94-8ac7-4f73-a0a8-51539dcc47d4] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance fc403a94-8ac7-4f73-a0a8-51539dcc47d4. [ 706.001095] nova-conductor[53894]: DEBUG nova.network.neutron [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] [instance: fc403a94-8ac7-4f73-a0a8-51539dcc47d4] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 706.064960] nova-conductor[53894]: DEBUG nova.network.neutron [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] [instance: fc403a94-8ac7-4f73-a0a8-51539dcc47d4] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 706.068178] nova-conductor[53894]: DEBUG nova.network.neutron [None req-6a1c52e1-25c8-4ed4-ba72-ee6743bfc1d9 tempest-ServerExternalEventsTest-1272541702 tempest-ServerExternalEventsTest-1272541702-project-member] [instance: fc403a94-8ac7-4f73-a0a8-51539dcc47d4] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 706.099108] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] [instance: bfa7d6f2-050a-4604-8f9c-250d89b091a5] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 75516f8b-27d7-431d-ac1d-969b78af4c51, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance bfa7d6f2-050a-4604-8f9c-250d89b091a5 was re-scheduled: Binding failed for port 75516f8b-27d7-431d-ac1d-969b78af4c51, please check neutron logs for more information.\n'] [ 706.099841] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 706.100206] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance bfa7d6f2-050a-4604-8f9c-250d89b091a5.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance bfa7d6f2-050a-4604-8f9c-250d89b091a5. [ 706.100591] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] [instance: bfa7d6f2-050a-4604-8f9c-250d89b091a5] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance bfa7d6f2-050a-4604-8f9c-250d89b091a5. [ 706.128558] nova-conductor[53893]: DEBUG nova.network.neutron [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] [instance: bfa7d6f2-050a-4604-8f9c-250d89b091a5] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 706.199244] nova-conductor[53893]: DEBUG nova.network.neutron [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] [instance: bfa7d6f2-050a-4604-8f9c-250d89b091a5] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 706.203713] nova-conductor[53893]: DEBUG nova.network.neutron [None req-708a8604-ec99-4065-8bc7-e241f7063359 tempest-TenantUsagesTestJSON-507762357 tempest-TenantUsagesTestJSON-507762357-project-member] [instance: bfa7d6f2-050a-4604-8f9c-250d89b091a5] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 708.927102] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Expecting reply to msg 2132e076f7c541aba23b5f5a99408699 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 709.110786] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2132e076f7c541aba23b5f5a99408699 [ 709.111628] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Took 0.19 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 709.132825] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 709.133106] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 709.133282] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 709.192492] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 709.192730] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 709.192899] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 709.193290] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 709.193459] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 709.193614] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 709.210919] nova-conductor[53893]: DEBUG nova.quota [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Getting quotas for project afc5e909ec5c4dd983ece5aa3236910f. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 709.214444] nova-conductor[53893]: DEBUG nova.quota [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Getting quotas for user ee5a21ff43314c1a857f6958056173f9 and project afc5e909ec5c4dd983ece5aa3236910f. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 709.220767] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 8f3227ba-f30a-4725-94d9-ac1d5f1f16de] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 709.221328] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 709.221532] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 709.221698] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 709.227616] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 8f3227ba-f30a-4725-94d9-ac1d5f1f16de] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 709.228091] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 709.228091] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 709.228300] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 709.251713] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 709.251713] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 709.251713] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 709.808618] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Expecting reply to msg d95a03c2d51a412280b2c8aef73e7e6d in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 710.008038] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d95a03c2d51a412280b2c8aef73e7e6d [ 710.008038] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Took 0.20 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 710.024652] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 710.025125] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 710.025125] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 710.076675] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 710.076969] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 710.077193] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 710.077602] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 710.078203] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 710.078203] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 710.089096] nova-conductor[53893]: DEBUG nova.quota [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Getting quotas for project 05666a0abf0b433184c14ff43e6e82c1. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 710.094375] nova-conductor[53893]: DEBUG nova.quota [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Getting quotas for user bfb9e86398ee44b7b9e06dcdc29f8ff6 and project 05666a0abf0b433184c14ff43e6e82c1. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 710.099163] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] [instance: b16d0a82-271e-4e37-bfcd-49c3749d16ac] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 710.099820] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 710.100100] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 710.100313] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 710.103691] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] [instance: b16d0a82-271e-4e37-bfcd-49c3749d16ac] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 710.104372] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 710.104564] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 710.104726] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 710.130216] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 710.130450] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 710.130623] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 713.267969] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell0:SELECT=15 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 713.340367] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Expecting reply to msg 201d2d68821c4eb8aa9b5553b0cc0d40 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 713.342349] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=71,nova_cell1:INSERT=160,nova_cell1:SAVEPOINT=7,nova_cell1:RELEASE=7 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 713.357214] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_api:SELECT=89,nova_api:DELETE=4,nova_api:UPDATE=6 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 713.369719] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_api:SELECT=101,nova_api:UPDATE=2,nova_api:DELETE=4 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 713.507312] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] [instance: deb53e97-7f20-47d3-a069-0e435776bad8] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 87352393-9957-4fc9-a5bc-009618ab2125, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance deb53e97-7f20-47d3-a069-0e435776bad8 was re-scheduled: Binding failed for port 87352393-9957-4fc9-a5bc-009618ab2125, please check neutron logs for more information.\n'] [ 713.508663] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 713.511233] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance deb53e97-7f20-47d3-a069-0e435776bad8.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance deb53e97-7f20-47d3-a069-0e435776bad8. [ 713.511635] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] [instance: deb53e97-7f20-47d3-a069-0e435776bad8] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance deb53e97-7f20-47d3-a069-0e435776bad8. [ 713.554721] nova-conductor[53893]: DEBUG nova.network.neutron [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] [instance: deb53e97-7f20-47d3-a069-0e435776bad8] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 713.563221] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 201d2d68821c4eb8aa9b5553b0cc0d40 [ 713.564068] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Took 0.23 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 713.580882] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 713.581148] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 713.581357] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 713.640505] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 713.640505] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 713.640650] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 713.641282] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 713.641282] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 713.641426] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 713.654994] nova-conductor[53893]: DEBUG nova.quota [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Getting quotas for project 8816e27bfe9140da98937bc2501fca42. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 713.659094] nova-conductor[53893]: DEBUG nova.quota [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Getting quotas for user 59cf18f84b8c41fbb06db5e1e948ae80 and project 8816e27bfe9140da98937bc2501fca42. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 713.665323] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] [instance: eb4ab6f2-3815-4bf1-a561-79bcdb74380c] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 713.665815] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 713.667988] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 713.667988] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 713.671415] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] [instance: eb4ab6f2-3815-4bf1-a561-79bcdb74380c] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 713.673179] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 713.673179] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 713.673179] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 713.694415] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 713.694536] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 713.694730] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 713.702043] nova-conductor[53893]: DEBUG nova.network.neutron [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] [instance: deb53e97-7f20-47d3-a069-0e435776bad8] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 713.706090] nova-conductor[53893]: DEBUG nova.network.neutron [None req-efdb9b24-4111-43e6-8816-67259c4f218c tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] [instance: deb53e97-7f20-47d3-a069-0e435776bad8] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 714.291352] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] [instance: e20b5017-bc1a-41c0-ba4b-83e4df43a53d] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port e0bc5458-78be-41d3-b081-0861882153df, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance e20b5017-bc1a-41c0-ba4b-83e4df43a53d was re-scheduled: Binding failed for port e0bc5458-78be-41d3-b081-0861882153df, please check neutron logs for more information.\n'] [ 714.292053] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 714.292298] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e20b5017-bc1a-41c0-ba4b-83e4df43a53d.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e20b5017-bc1a-41c0-ba4b-83e4df43a53d. [ 714.292623] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] [instance: e20b5017-bc1a-41c0-ba4b-83e4df43a53d] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e20b5017-bc1a-41c0-ba4b-83e4df43a53d. [ 714.341130] nova-conductor[53893]: DEBUG nova.network.neutron [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] [instance: e20b5017-bc1a-41c0-ba4b-83e4df43a53d] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 714.475701] nova-conductor[53893]: DEBUG nova.network.neutron [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] [instance: e20b5017-bc1a-41c0-ba4b-83e4df43a53d] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 714.485439] nova-conductor[53893]: DEBUG nova.network.neutron [None req-a46ce3f8-f57d-469c-a545-bbe150e6a998 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] [instance: e20b5017-bc1a-41c0-ba4b-83e4df43a53d] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 715.296204] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] [instance: 151fefe2-b70a-4ea5-8b50-08c7968b10fe] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn\n self._fetch_image_if_missing(context, vi)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing\n image_cache(vi, tmp_image_ds_loc)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image\n vm_util.copy_virtual_disk(\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk\n session._wait_for_task(vmdk_copy_task)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task\n return self.wait_for_task(task_ref)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task\n return evt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner\n self.f(*self.args, **self.kw)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task\n raise exceptions.translate_fault(task_info.error)\n', "oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n", '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', "nova.exception.RescheduledException: Build of instance 151fefe2-b70a-4ea5-8b50-08c7968b10fe was re-scheduled: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n"] [ 715.296769] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 715.297736] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 151fefe2-b70a-4ea5-8b50-08c7968b10fe.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 151fefe2-b70a-4ea5-8b50-08c7968b10fe. [ 715.298044] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-573cc642-9354-49ea-8033-f5044093c26e tempest-ServerDiagnosticsV248Test-2019202710 tempest-ServerDiagnosticsV248Test-2019202710-project-member] [instance: 151fefe2-b70a-4ea5-8b50-08c7968b10fe] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 151fefe2-b70a-4ea5-8b50-08c7968b10fe. [ 717.370015] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] [instance: 7b2debf8-278d-443f-aaf7-3ae6c129981c] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port aeedbef6-9d53-4e69-9e27-8281b1f12959, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 7b2debf8-278d-443f-aaf7-3ae6c129981c was re-scheduled: Binding failed for port aeedbef6-9d53-4e69-9e27-8281b1f12959, please check neutron logs for more information.\n'] [ 717.370650] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 717.374152] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7b2debf8-278d-443f-aaf7-3ae6c129981c.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7b2debf8-278d-443f-aaf7-3ae6c129981c. [ 717.374152] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] [instance: 7b2debf8-278d-443f-aaf7-3ae6c129981c] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7b2debf8-278d-443f-aaf7-3ae6c129981c. [ 717.406684] nova-conductor[53893]: DEBUG nova.network.neutron [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] [instance: 7b2debf8-278d-443f-aaf7-3ae6c129981c] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 717.641602] nova-conductor[53893]: DEBUG nova.network.neutron [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] [instance: 7b2debf8-278d-443f-aaf7-3ae6c129981c] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 717.650979] nova-conductor[53893]: DEBUG nova.network.neutron [None req-64b5fb14-0080-49e2-98ce-dd286cdd12b6 tempest-ServersTestManualDisk-40187549 tempest-ServersTestManualDisk-40187549-project-member] [instance: 7b2debf8-278d-443f-aaf7-3ae6c129981c] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 717.836094] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Expecting reply to msg 77d51e26a0574667a23f8da600d11286 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 718.042016] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 77d51e26a0574667a23f8da600d11286 [ 718.042016] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Took 0.22 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 718.074300] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 718.075605] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 718.075605] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 718.095356] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] [instance: b3e22378-e257-4e49-9eb2-787b4afd0eb8] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 6f14da6b-a787-45eb-a5ad-99eec27448be, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance b3e22378-e257-4e49-9eb2-787b4afd0eb8 was re-scheduled: Binding failed for port 6f14da6b-a787-45eb-a5ad-99eec27448be, please check neutron logs for more information.\n'] [ 718.095356] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 718.095356] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b3e22378-e257-4e49-9eb2-787b4afd0eb8.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b3e22378-e257-4e49-9eb2-787b4afd0eb8. [ 718.095356] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] [instance: b3e22378-e257-4e49-9eb2-787b4afd0eb8] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b3e22378-e257-4e49-9eb2-787b4afd0eb8. [ 718.136548] nova-conductor[53894]: DEBUG nova.network.neutron [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] [instance: b3e22378-e257-4e49-9eb2-787b4afd0eb8] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 718.151576] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 718.151913] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 718.153094] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 718.153094] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 718.153094] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 718.153094] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 718.170235] nova-conductor[53894]: DEBUG nova.quota [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Getting quotas for project 3dfb77f12805418eaa6127fc75becec0. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 718.176730] nova-conductor[53894]: DEBUG nova.quota [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Getting quotas for user 789177a2f7be455cadec45cf03d67521 and project 3dfb77f12805418eaa6127fc75becec0. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 718.184713] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: b465e4bb-f9f4-4fcf-97e7-ad38f33918a4] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 718.185185] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 718.185384] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 718.185550] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 718.190441] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: b465e4bb-f9f4-4fcf-97e7-ad38f33918a4] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 718.191338] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 718.191338] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 718.191521] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 718.200440] nova-conductor[53894]: DEBUG nova.network.neutron [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] [instance: b3e22378-e257-4e49-9eb2-787b4afd0eb8] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 718.204466] nova-conductor[53894]: DEBUG nova.network.neutron [None req-b44752ba-6ef4-4792-b6dc-ca0cc2296584 tempest-ServerRescueTestJSONUnderV235-863282302 tempest-ServerRescueTestJSONUnderV235-863282302-project-member] [instance: b3e22378-e257-4e49-9eb2-787b4afd0eb8] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 718.224332] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 718.224332] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 718.224332] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 720.589572] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Expecting reply to msg 24e1c3e607a044039b564d8cf09c5328 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 720.740880] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 24e1c3e607a044039b564d8cf09c5328 [ 720.741808] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Took 0.16 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 720.756609] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 720.757300] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 720.757300] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 720.817311] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 720.817311] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 720.817311] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 720.817311] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 720.817599] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 720.818234] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 720.831112] nova-conductor[53894]: DEBUG nova.quota [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Getting quotas for project b239a909024e4ca698dab05075729b28. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 720.835964] nova-conductor[53894]: DEBUG nova.quota [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Getting quotas for user 7b226f922a774dbba9acbc3b01ebaf3f and project b239a909024e4ca698dab05075729b28. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 720.844167] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] [instance: fd8bf9dd-bed4-4f79-9d5b-80073b19649a] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 720.844687] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 720.844897] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 720.845123] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 720.850781] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] [instance: fd8bf9dd-bed4-4f79-9d5b-80073b19649a] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 720.851547] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 720.851756] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 720.851965] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 720.875569] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 720.875693] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 720.875749] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 723.371298] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Expecting reply to msg 45da002be01e4bdc9b3a09abf2a37a68 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 723.527777] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 45da002be01e4bdc9b3a09abf2a37a68 [ 723.528614] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Took 0.16 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 723.549314] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 723.549547] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 723.549720] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 723.592629] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 723.592629] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 723.592720] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 723.593399] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 723.593399] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 723.593399] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 723.606808] nova-conductor[53893]: DEBUG nova.quota [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Getting quotas for project 040cba7015c544f0a1dcc43e37b20351. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 723.609132] nova-conductor[53893]: DEBUG nova.quota [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Getting quotas for user 50be22513aa7452db8ce44f19f0ba863 and project 040cba7015c544f0a1dcc43e37b20351. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 723.616196] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] [instance: 59555974-e1a8-467a-872e-5d0cd4daffdb] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 723.616663] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 723.617362] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 723.617547] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 723.622471] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] [instance: 59555974-e1a8-467a-872e-5d0cd4daffdb] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 723.623109] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 723.623319] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 723.623490] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 723.654653] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 723.655334] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 723.655483] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 725.231021] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=892,nova_cell1:UPDATE=251,nova_cell1:SAVEPOINT=51,nova_cell1:RELEASE=51,nova_cell1:INSERT=43 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 725.673350] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 8f3227ba-f30a-4725-94d9-ac1d5f1f16de] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port e317892b-9c88-4df9-a67b-0acebb4686d3, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 8f3227ba-f30a-4725-94d9-ac1d5f1f16de was re-scheduled: Binding failed for port e317892b-9c88-4df9-a67b-0acebb4686d3, please check neutron logs for more information.\n'] [ 725.674105] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 725.674355] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8f3227ba-f30a-4725-94d9-ac1d5f1f16de.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8f3227ba-f30a-4725-94d9-ac1d5f1f16de. [ 725.674684] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 8f3227ba-f30a-4725-94d9-ac1d5f1f16de] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8f3227ba-f30a-4725-94d9-ac1d5f1f16de. [ 725.704842] nova-conductor[53894]: DEBUG nova.network.neutron [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 8f3227ba-f30a-4725-94d9-ac1d5f1f16de] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 725.812633] nova-conductor[53894]: DEBUG nova.network.neutron [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 8f3227ba-f30a-4725-94d9-ac1d5f1f16de] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 725.820164] nova-conductor[53894]: DEBUG nova.network.neutron [None req-45c2815a-b388-4710-b579-e9e5ab727b2e tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 8f3227ba-f30a-4725-94d9-ac1d5f1f16de] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 726.624335] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] [instance: b16d0a82-271e-4e37-bfcd-49c3749d16ac] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 73cedda1-f0ab-4dab-88e0-e194b3a607c2, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance b16d0a82-271e-4e37-bfcd-49c3749d16ac was re-scheduled: Binding failed for port 73cedda1-f0ab-4dab-88e0-e194b3a607c2, please check neutron logs for more information.\n'] [ 726.624335] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 726.624335] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b16d0a82-271e-4e37-bfcd-49c3749d16ac.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b16d0a82-271e-4e37-bfcd-49c3749d16ac. [ 726.624734] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] [instance: b16d0a82-271e-4e37-bfcd-49c3749d16ac] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b16d0a82-271e-4e37-bfcd-49c3749d16ac. [ 726.657224] nova-conductor[53893]: DEBUG nova.network.neutron [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] [instance: b16d0a82-271e-4e37-bfcd-49c3749d16ac] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 726.753731] nova-conductor[53893]: DEBUG nova.network.neutron [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] [instance: b16d0a82-271e-4e37-bfcd-49c3749d16ac] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 726.759026] nova-conductor[53893]: DEBUG nova.network.neutron [None req-3adae638-c63b-4063-983a-6aa05bac6f52 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] [instance: b16d0a82-271e-4e37-bfcd-49c3749d16ac] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 729.297717] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Expecting reply to msg 795b74717fa847c9b5bbbfc3ada026a6 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 729.478460] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 795b74717fa847c9b5bbbfc3ada026a6 [ 729.479296] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Took 0.18 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 729.496232] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 729.496468] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 729.496639] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 729.545065] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 729.545305] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 729.545478] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 729.545847] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 729.546040] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 729.546210] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 729.556214] nova-conductor[53893]: DEBUG nova.quota [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Getting quotas for project 7097bed89f7c4ec699866cb25bf49e47. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 729.559736] nova-conductor[53893]: DEBUG nova.quota [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Getting quotas for user 6189f904167c4cfa9fe287573713f17b and project 7097bed89f7c4ec699866cb25bf49e47. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 729.565157] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] [instance: 578faa1c-9edd-4ce3-8a5d-add49367d390] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 729.565584] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 729.565785] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 729.565951] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 729.570305] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] [instance: 578faa1c-9edd-4ce3-8a5d-add49367d390] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 729.571013] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 729.571220] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 729.571391] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 729.588721] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 729.588949] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 729.589138] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 730.821587] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell0:SELECT=4 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 730.854798] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=18,nova_cell1:INSERT=39,nova_cell1:SAVEPOINT=1,nova_cell1:RELEASE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 730.868957] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_api:SELECT=19,nova_api:UPDATE=2,nova_api:DELETE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 730.870802] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_api:SELECT=29,nova_api:DELETE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 731.554029] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=894,nova_cell1:UPDATE=250,nova_cell1:SAVEPOINT=48,nova_cell1:RELEASE=48,nova_cell1:INSERT=31 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 731.801941] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Expecting reply to msg 9511780ba53c4fa8a304dc2efa1f6883 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 731.990875] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 9511780ba53c4fa8a304dc2efa1f6883 [ 731.991774] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Took 0.19 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 732.015185] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 732.015400] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 732.015574] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 732.057104] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 732.057345] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 732.057518] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 732.057899] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 732.058124] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 732.058297] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 732.066568] nova-conductor[53893]: DEBUG nova.quota [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Getting quotas for project 9dbc2efe50ed4ae5a5f0cf6f492a20bf. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 732.069476] nova-conductor[53893]: DEBUG nova.quota [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Getting quotas for user 7ce2159a0ccb46c89a7574d04142e926 and project 9dbc2efe50ed4ae5a5f0cf6f492a20bf. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 732.075807] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: e5d9ba62-f701-4c6a-8dbe-1bd401db3343] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 732.076288] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 732.076491] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 732.076657] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 732.081145] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: e5d9ba62-f701-4c6a-8dbe-1bd401db3343] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 732.081785] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 732.082015] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 732.082164] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 732.098975] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 732.099228] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 732.099400] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 733.250526] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] [instance: eb4ab6f2-3815-4bf1-a561-79bcdb74380c] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 9d511783-4632-4be0-822a-27f7347d9f78, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance eb4ab6f2-3815-4bf1-a561-79bcdb74380c was re-scheduled: Binding failed for port 9d511783-4632-4be0-822a-27f7347d9f78, please check neutron logs for more information.\n'] [ 733.251305] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 733.251581] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance eb4ab6f2-3815-4bf1-a561-79bcdb74380c.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance eb4ab6f2-3815-4bf1-a561-79bcdb74380c. [ 733.251803] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] [instance: eb4ab6f2-3815-4bf1-a561-79bcdb74380c] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance eb4ab6f2-3815-4bf1-a561-79bcdb74380c. [ 733.279012] nova-conductor[53894]: DEBUG nova.network.neutron [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] [instance: eb4ab6f2-3815-4bf1-a561-79bcdb74380c] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 733.463111] nova-conductor[53894]: DEBUG nova.network.neutron [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] [instance: eb4ab6f2-3815-4bf1-a561-79bcdb74380c] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 733.468298] nova-conductor[53894]: DEBUG nova.network.neutron [None req-0d1a2f85-ac2e-4917-be3f-f8e547a4224d tempest-InstanceActionsTestJSON-1806126258 tempest-InstanceActionsTestJSON-1806126258-project-member] [instance: eb4ab6f2-3815-4bf1-a561-79bcdb74380c] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 734.987243] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Expecting reply to msg 7de196af5adf4524baee8a707bb12df4 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 735.163282] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7de196af5adf4524baee8a707bb12df4 [ 735.164117] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Took 0.18 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 735.186852] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.187262] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.188914] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.241886] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.241886] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.241886] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.241886] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.241886] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.241886] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.249617] nova-conductor[53894]: DEBUG nova.quota [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Getting quotas for project 7097bed89f7c4ec699866cb25bf49e47. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 735.253118] nova-conductor[53894]: DEBUG nova.quota [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Getting quotas for user 6189f904167c4cfa9fe287573713f17b and project 7097bed89f7c4ec699866cb25bf49e47. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 735.258954] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] [instance: cabdd4fe-9515-45f0-b596-862986f5733e] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 735.259477] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.259750] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.259967] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.271774] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] [instance: cabdd4fe-9515-45f0-b596-862986f5733e] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 735.271774] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.271774] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.271774] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 735.306612] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 735.306612] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 735.306612] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 736.078378] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: b465e4bb-f9f4-4fcf-97e7-ad38f33918a4] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 66c001d7-317a-4fa7-9e60-ec7751d386f5, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance b465e4bb-f9f4-4fcf-97e7-ad38f33918a4 was re-scheduled: Binding failed for port 66c001d7-317a-4fa7-9e60-ec7751d386f5, please check neutron logs for more information.\n'] [ 736.079083] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 736.079297] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b465e4bb-f9f4-4fcf-97e7-ad38f33918a4.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b465e4bb-f9f4-4fcf-97e7-ad38f33918a4. [ 736.079861] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: b465e4bb-f9f4-4fcf-97e7-ad38f33918a4] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b465e4bb-f9f4-4fcf-97e7-ad38f33918a4. [ 736.126142] nova-conductor[53894]: DEBUG nova.network.neutron [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: b465e4bb-f9f4-4fcf-97e7-ad38f33918a4] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 736.245040] nova-conductor[53894]: DEBUG nova.network.neutron [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: b465e4bb-f9f4-4fcf-97e7-ad38f33918a4] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 736.248627] nova-conductor[53894]: DEBUG nova.network.neutron [None req-a883ff76-8a52-41f9-815d-cb464c39e058 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: b465e4bb-f9f4-4fcf-97e7-ad38f33918a4] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 737.866449] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Expecting reply to msg fb27ff11306e4ed9bb9c7da4e8dcadfe in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 738.095105] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fb27ff11306e4ed9bb9c7da4e8dcadfe [ 738.096414] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Took 0.24 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 738.117067] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 738.117514] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 738.117772] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 738.161550] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 738.162343] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 738.162343] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 738.162546] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 738.162787] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 738.163148] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 738.176795] nova-conductor[53894]: DEBUG nova.quota [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Getting quotas for project 57787a709d744ea4a19a2cfb923d89d4. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 738.180744] nova-conductor[53894]: DEBUG nova.quota [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Getting quotas for user 2d6b88b1f4394a63b32be00fc9dcc761 and project 57787a709d744ea4a19a2cfb923d89d4. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 738.189722] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: a2429a5f-fe61-46b9-a71d-a6ddd62a6e08] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 738.190374] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 738.190565] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 738.191744] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 738.203065] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: a2429a5f-fe61-46b9-a71d-a6ddd62a6e08] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 738.203841] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 738.204070] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 738.204288] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 738.231803] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 738.232120] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 738.232382] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 739.585874] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] [instance: fd8bf9dd-bed4-4f79-9d5b-80073b19649a] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 7cc4bed0-779d-4ad1-b83a-227f0c2229e4, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance fd8bf9dd-bed4-4f79-9d5b-80073b19649a was re-scheduled: Binding failed for port 7cc4bed0-779d-4ad1-b83a-227f0c2229e4, please check neutron logs for more information.\n'] [ 739.588109] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 739.588776] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance fd8bf9dd-bed4-4f79-9d5b-80073b19649a.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance fd8bf9dd-bed4-4f79-9d5b-80073b19649a. [ 739.590465] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] [instance: fd8bf9dd-bed4-4f79-9d5b-80073b19649a] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance fd8bf9dd-bed4-4f79-9d5b-80073b19649a. [ 739.621409] nova-conductor[53894]: DEBUG nova.network.neutron [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] [instance: fd8bf9dd-bed4-4f79-9d5b-80073b19649a] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 739.713030] nova-conductor[53894]: DEBUG nova.network.neutron [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] [instance: fd8bf9dd-bed4-4f79-9d5b-80073b19649a] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 739.720996] nova-conductor[53894]: DEBUG nova.network.neutron [None req-4419b8b7-7473-4213-9100-092ec2267cc4 tempest-ImagesNegativeTestJSON-484615250 tempest-ImagesNegativeTestJSON-484615250-project-member] [instance: fd8bf9dd-bed4-4f79-9d5b-80073b19649a] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 740.545438] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Expecting reply to msg 0810499e9ee64b3686f71891cda712e9 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 740.683221] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 0810499e9ee64b3686f71891cda712e9 [ 740.683221] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Took 0.14 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 740.700704] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 740.703051] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 740.703051] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 740.745360] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 740.746611] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.002s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 740.746816] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 740.747232] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 740.747413] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 740.747570] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 740.763699] nova-conductor[53893]: DEBUG nova.quota [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Getting quotas for project afc5e909ec5c4dd983ece5aa3236910f. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 740.767141] nova-conductor[53893]: DEBUG nova.quota [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Getting quotas for user ee5a21ff43314c1a857f6958056173f9 and project afc5e909ec5c4dd983ece5aa3236910f. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 740.776762] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 5efb5557-deab-4eec-ac5a-7aabb4477c61] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 740.776762] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 740.776762] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 740.776762] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 740.780273] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 5efb5557-deab-4eec-ac5a-7aabb4477c61] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 740.780940] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 740.781229] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 740.781891] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 740.801055] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 740.801331] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 740.801532] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 741.696923] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] [instance: 59555974-e1a8-467a-872e-5d0cd4daffdb] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port fc83359a-f7b2-4614-bff2-cd718a34a194, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 59555974-e1a8-467a-872e-5d0cd4daffdb was re-scheduled: Binding failed for port fc83359a-f7b2-4614-bff2-cd718a34a194, please check neutron logs for more information.\n'] [ 741.697575] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 741.698784] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 59555974-e1a8-467a-872e-5d0cd4daffdb.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 59555974-e1a8-467a-872e-5d0cd4daffdb. [ 741.698784] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] [instance: 59555974-e1a8-467a-872e-5d0cd4daffdb] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 59555974-e1a8-467a-872e-5d0cd4daffdb. [ 741.733403] nova-conductor[53893]: DEBUG nova.network.neutron [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] [instance: 59555974-e1a8-467a-872e-5d0cd4daffdb] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 741.883319] nova-conductor[53893]: DEBUG nova.network.neutron [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] [instance: 59555974-e1a8-467a-872e-5d0cd4daffdb] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 741.888348] nova-conductor[53893]: DEBUG nova.network.neutron [None req-b9a97487-71d8-415d-99f1-92cac8219fa5 tempest-ServersNegativeTestJSON-52319824 tempest-ServersNegativeTestJSON-52319824-project-member] [instance: 59555974-e1a8-467a-872e-5d0cd4daffdb] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 743.952604] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Expecting reply to msg b52ad44feaf44791a9efb7beb073a027 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 743.958960] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Expecting reply to msg da67ee76e2a64968a187a3b77faaafe1 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 744.121664] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b52ad44feaf44791a9efb7beb073a027 [ 744.123704] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Took 0.17 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 744.142188] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg da67ee76e2a64968a187a3b77faaafe1 [ 744.143324] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Took 0.19 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 744.146581] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 744.147375] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 744.149448] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 744.160016] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 744.160016] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 744.160161] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 744.196895] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 744.197148] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 744.197332] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 744.197744] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 744.197818] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 744.197982] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 744.198212] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 744.198455] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 744.198885] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 744.199139] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 744.199430] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 744.199589] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 744.210271] nova-conductor[53893]: DEBUG nova.quota [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Getting quotas for project 4455e9f03bab49eba3d941e3565381cc. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 744.212936] nova-conductor[53893]: DEBUG nova.quota [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Getting quotas for user b20e19f3ac704956aae925f564d4b286 and project 4455e9f03bab49eba3d941e3565381cc. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 744.213684] nova-conductor[53894]: DEBUG nova.quota [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Getting quotas for project 3dfb77f12805418eaa6127fc75becec0. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 744.218214] nova-conductor[53894]: DEBUG nova.quota [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Getting quotas for user 789177a2f7be455cadec45cf03d67521 and project 3dfb77f12805418eaa6127fc75becec0. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 744.219129] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] [instance: 807b279c-5934-4b4b-977d-c02a8dcbbdb6] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 744.220039] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 744.220222] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 744.220409] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 744.224823] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 51ae3e83-1cf3-4f56-b48d-b436ae84d706] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 744.225312] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 744.225951] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 744.225951] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 744.229328] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] [instance: 807b279c-5934-4b4b-977d-c02a8dcbbdb6] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 744.230163] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 744.230580] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 744.230645] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 744.240526] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 51ae3e83-1cf3-4f56-b48d-b436ae84d706] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 744.242144] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 744.242144] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 744.242144] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 744.265971] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 744.266961] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 744.267266] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 744.275229] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 744.275229] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 744.275229] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 746.589369] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Expecting reply to msg ccdfd5c1cd154941bf8f8c29c84aeb84 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 746.762484] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ccdfd5c1cd154941bf8f8c29c84aeb84 [ 746.763331] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Took 0.18 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 746.780300] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 746.780300] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 746.780300] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 746.858369] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 746.858595] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 746.858765] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 746.859143] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 746.859333] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 746.859494] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 746.876958] nova-conductor[53894]: DEBUG nova.quota [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Getting quotas for project 3c37c09a0a33411f8aa3240ac2adcff5. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 746.883585] nova-conductor[53894]: DEBUG nova.quota [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Getting quotas for user 1aeb3b98fa3b4e81845ed61d742f6258 and project 3c37c09a0a33411f8aa3240ac2adcff5. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 746.889435] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] [instance: 681fb733-cd37-4f73-a487-e4856206907f] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 746.892847] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 746.892847] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 746.892847] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 746.897519] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] [instance: 681fb733-cd37-4f73-a487-e4856206907f] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 746.898254] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 746.898484] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 746.898636] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 746.926418] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 746.926653] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 746.926872] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 747.916103] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Expecting reply to msg c4d98a31911048998fed8f62d66c8aa3 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 748.086589] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c4d98a31911048998fed8f62d66c8aa3 [ 748.087231] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Took 0.17 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 748.103432] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 748.103796] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 748.103894] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 748.182685] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 748.183013] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 748.183334] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 748.183841] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 748.184072] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 748.184248] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 748.202340] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] [instance: 578faa1c-9edd-4ce3-8a5d-add49367d390] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port b5fdbb40-b9cc-4ce3-b78b-ae060cf5a981, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 578faa1c-9edd-4ce3-8a5d-add49367d390 was re-scheduled: Binding failed for port b5fdbb40-b9cc-4ce3-b78b-ae060cf5a981, please check neutron logs for more information.\n'] [ 748.202921] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 748.203169] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 578faa1c-9edd-4ce3-8a5d-add49367d390.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 578faa1c-9edd-4ce3-8a5d-add49367d390. [ 748.203480] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] [instance: 578faa1c-9edd-4ce3-8a5d-add49367d390] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 578faa1c-9edd-4ce3-8a5d-add49367d390. [ 748.208346] nova-conductor[53893]: DEBUG nova.quota [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Getting quotas for project bff3be1976444e58a2b7be93d47f50ef. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 748.220019] nova-conductor[53893]: DEBUG nova.quota [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Getting quotas for user 84f05a61e0f6499bb35c44d254226249 and project bff3be1976444e58a2b7be93d47f50ef. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 748.231334] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 45a40160-c224-4a8f-8e92-26d770d4ff4f] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 748.232119] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 748.232357] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 748.232557] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 748.237810] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 45a40160-c224-4a8f-8e92-26d770d4ff4f] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 748.238473] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 748.238681] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 748.238848] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 748.240482] nova-conductor[53893]: DEBUG nova.network.neutron [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] [instance: 578faa1c-9edd-4ce3-8a5d-add49367d390] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 748.265633] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 748.265941] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 748.266150] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 748.318113] nova-conductor[53893]: DEBUG nova.network.neutron [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] [instance: 578faa1c-9edd-4ce3-8a5d-add49367d390] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 748.325577] nova-conductor[53893]: DEBUG nova.network.neutron [None req-cf61da20-a151-401c-941c-0cce708baa2b tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] [instance: 578faa1c-9edd-4ce3-8a5d-add49367d390] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 748.981015] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Expecting reply to msg f84a01c106484a589fb1204a1bea72f7 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 749.152582] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f84a01c106484a589fb1204a1bea72f7 [ 749.153666] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Took 0.18 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 749.157613] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=130,nova_api:UPDATE=8,nova_api:DELETE=4 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 749.158183] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=138,nova_api:UPDATE=3,nova_api:DELETE=7 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 749.179017] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 749.179278] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 749.179452] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 749.182628] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=100,nova_cell1:SAVEPOINT=9,nova_cell1:INSERT=218,nova_cell1:RELEASE=9 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 749.236274] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 749.239393] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 749.239393] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 749.239393] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 749.239393] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 749.239393] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 749.245092] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell0:SELECT=23 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 749.255910] nova-conductor[53893]: DEBUG nova.quota [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Getting quotas for project a0f89ca420ba4732b58127285d726cb2. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 749.259268] nova-conductor[53893]: DEBUG nova.quota [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Getting quotas for user 6f35b9f152d5463797ee9ca40e847a12 and project a0f89ca420ba4732b58127285d726cb2. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 749.266340] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] [instance: d16dff71-2dab-469a-8cb1-40ed086c42d1] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 749.266877] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 749.267717] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 749.267717] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 749.273987] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] [instance: d16dff71-2dab-469a-8cb1-40ed086c42d1] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 749.274810] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 749.275043] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 749.275224] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 749.295018] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 749.295018] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 749.295018] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 749.914900] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: e5d9ba62-f701-4c6a-8dbe-1bd401db3343] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 8cd305c7-8efe-43f6-b7a1-97c329a3b663, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance e5d9ba62-f701-4c6a-8dbe-1bd401db3343 was re-scheduled: Binding failed for port 8cd305c7-8efe-43f6-b7a1-97c329a3b663, please check neutron logs for more information.\n'] [ 749.914900] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 749.914900] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e5d9ba62-f701-4c6a-8dbe-1bd401db3343.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e5d9ba62-f701-4c6a-8dbe-1bd401db3343. [ 749.914900] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: e5d9ba62-f701-4c6a-8dbe-1bd401db3343] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e5d9ba62-f701-4c6a-8dbe-1bd401db3343. [ 749.943574] nova-conductor[53893]: DEBUG nova.network.neutron [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: e5d9ba62-f701-4c6a-8dbe-1bd401db3343] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 750.066465] nova-conductor[53893]: DEBUG nova.network.neutron [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: e5d9ba62-f701-4c6a-8dbe-1bd401db3343] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 750.071204] nova-conductor[53893]: DEBUG nova.network.neutron [None req-f29ed5bc-91ef-4153-adc4-2a9a1cf4f3ca tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: e5d9ba62-f701-4c6a-8dbe-1bd401db3343] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 752.288827] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Expecting reply to msg e1074bb42d844948b99f86fda8e1ce4a in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 752.478339] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e1074bb42d844948b99f86fda8e1ce4a [ 752.479774] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Took 0.19 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 752.504632] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.505708] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.505708] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.558249] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.558249] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.558249] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.558596] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.562019] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.562019] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.592424] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] [instance: cabdd4fe-9515-45f0-b596-862986f5733e] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 3499cf5d-3bf9-4c03-8f3d-25fd14830733, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance cabdd4fe-9515-45f0-b596-862986f5733e was re-scheduled: Binding failed for port 3499cf5d-3bf9-4c03-8f3d-25fd14830733, please check neutron logs for more information.\n'] [ 752.593016] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 752.593266] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance cabdd4fe-9515-45f0-b596-862986f5733e.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance cabdd4fe-9515-45f0-b596-862986f5733e. [ 752.593556] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] [instance: cabdd4fe-9515-45f0-b596-862986f5733e] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance cabdd4fe-9515-45f0-b596-862986f5733e. [ 752.616934] nova-conductor[53894]: DEBUG nova.quota [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Getting quotas for project 79a9ed8a5ae24f64a929062d2664a4c9. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 752.623332] nova-conductor[53894]: DEBUG nova.quota [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Getting quotas for user 0697390bff094da5b8ec0fbb9ce233a8 and project 79a9ed8a5ae24f64a929062d2664a4c9. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 752.633389] nova-conductor[53894]: DEBUG nova.network.neutron [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] [instance: cabdd4fe-9515-45f0-b596-862986f5733e] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 752.636989] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] [instance: c588118f-5b15-4262-8d21-c4fbc3088f1f] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 752.636989] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.636989] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.637196] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.641382] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] [instance: c588118f-5b15-4262-8d21-c4fbc3088f1f] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 752.641948] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.642167] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.642346] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.660691] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 752.660841] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 752.661229] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 752.732606] nova-conductor[53894]: DEBUG nova.network.neutron [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] [instance: cabdd4fe-9515-45f0-b596-862986f5733e] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 752.736334] nova-conductor[53894]: DEBUG nova.network.neutron [None req-c9b99250-a19a-4128-b657-87181a2897a6 tempest-ListImageFiltersTestJSON-328745172 tempest-ListImageFiltersTestJSON-328745172-project-member] [instance: cabdd4fe-9515-45f0-b596-862986f5733e] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 753.680647] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: a2429a5f-fe61-46b9-a71d-a6ddd62a6e08] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port d4b27b9b-cbb6-4137-9f20-14a7edbbb116, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance a2429a5f-fe61-46b9-a71d-a6ddd62a6e08 was re-scheduled: Binding failed for port d4b27b9b-cbb6-4137-9f20-14a7edbbb116, please check neutron logs for more information.\n'] [ 753.681342] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 753.681608] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance a2429a5f-fe61-46b9-a71d-a6ddd62a6e08.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance a2429a5f-fe61-46b9-a71d-a6ddd62a6e08. [ 753.681940] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: a2429a5f-fe61-46b9-a71d-a6ddd62a6e08] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance a2429a5f-fe61-46b9-a71d-a6ddd62a6e08. [ 753.722857] nova-conductor[53894]: DEBUG nova.network.neutron [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: a2429a5f-fe61-46b9-a71d-a6ddd62a6e08] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 753.815466] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Expecting reply to msg 20251f84d0524fedb2bcff3e5b7efcc4 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 753.817432] nova-conductor[53894]: DEBUG nova.network.neutron [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: a2429a5f-fe61-46b9-a71d-a6ddd62a6e08] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 753.825016] nova-conductor[53894]: DEBUG nova.network.neutron [None req-a8e543f2-be34-4fba-842c-908ed995d413 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: a2429a5f-fe61-46b9-a71d-a6ddd62a6e08] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 753.985360] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 20251f84d0524fedb2bcff3e5b7efcc4 [ 753.985360] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Took 0.17 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 754.005018] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 754.005018] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 754.005018] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 754.039649] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 754.040172] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 754.040532] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 754.041352] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 754.041810] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 754.042173] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 754.052838] nova-conductor[53894]: DEBUG nova.quota [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Getting quotas for project 19b2170b4bde4207b5a774f1cd266796. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 754.057178] nova-conductor[53894]: DEBUG nova.quota [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Getting quotas for user 38ed2b39a2d0462095de83291bca4587 and project 19b2170b4bde4207b5a774f1cd266796. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 754.065325] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] [instance: de519927-4dbd-4e6b-ba0a-d02d01950749] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 754.065325] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 754.065325] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 754.065325] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 754.071399] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] [instance: de519927-4dbd-4e6b-ba0a-d02d01950749] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 754.072206] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 754.072454] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 754.072628] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 754.094365] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 754.094551] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 754.094725] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 755.492765] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Expecting reply to msg ade55a8cbc5d455fbcc653f79ea1af81 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 755.689811] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg ade55a8cbc5d455fbcc653f79ea1af81 [ 755.692138] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Took 0.20 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 755.715631] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 755.715631] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 755.715631] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 755.787555] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 755.787848] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 755.788054] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 755.788871] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 755.788871] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 755.788871] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 755.798596] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 5efb5557-deab-4eec-ac5a-7aabb4477c61] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 5084e7ba-5652-4703-8329-ca8f37da9c52, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 5efb5557-deab-4eec-ac5a-7aabb4477c61 was re-scheduled: Binding failed for port 5084e7ba-5652-4703-8329-ca8f37da9c52, please check neutron logs for more information.\n'] [ 755.798596] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 755.798753] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 5efb5557-deab-4eec-ac5a-7aabb4477c61.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 5efb5557-deab-4eec-ac5a-7aabb4477c61. [ 755.798975] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 5efb5557-deab-4eec-ac5a-7aabb4477c61] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 5efb5557-deab-4eec-ac5a-7aabb4477c61. [ 755.840075] nova-conductor[53893]: DEBUG nova.quota [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Getting quotas for project b54f5acf42c64133afbb208929492c31. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 755.846208] nova-conductor[53893]: DEBUG nova.quota [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Getting quotas for user 06291742abbe4dee8092fba657b8ab91 and project b54f5acf42c64133afbb208929492c31. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 755.852998] nova-conductor[53893]: DEBUG nova.network.neutron [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 5efb5557-deab-4eec-ac5a-7aabb4477c61] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 755.858736] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 3d6ead4a-228d-48da-89b3-4fa99f6299d2] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 755.859340] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 755.859390] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 755.859557] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 755.862638] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 3d6ead4a-228d-48da-89b3-4fa99f6299d2] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 755.863269] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 755.864707] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 755.864707] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 755.886683] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 755.888462] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 755.888462] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 756.008663] nova-conductor[53893]: DEBUG nova.network.neutron [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 5efb5557-deab-4eec-ac5a-7aabb4477c61] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 756.014301] nova-conductor[53893]: DEBUG nova.network.neutron [None req-af44bf33-46f6-40c1-8f64-f891e9f36c52 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 5efb5557-deab-4eec-ac5a-7aabb4477c61] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 757.222645] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Expecting reply to msg d9719a7e9c394cddab3d1082a9155bf3 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 757.229830] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Expecting reply to msg 02268f52e8984bcfa779d1044fe8a2f4 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 757.384207] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 02268f52e8984bcfa779d1044fe8a2f4 [ 757.385267] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Took 0.16 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 757.397516] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d9719a7e9c394cddab3d1082a9155bf3 [ 757.398617] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Took 0.18 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 757.407421] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.410098] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 757.410098] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 757.416689] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.417301] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 757.417543] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 757.466827] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.467527] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 757.467961] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 757.468459] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.468689] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 757.468823] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 757.474528] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.474955] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 757.475042] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 757.475417] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.475638] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 757.475823] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 757.486537] nova-conductor[53894]: DEBUG nova.quota [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Getting quotas for project 0ac54b53da9b4c9d99b34169cc521682. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 757.487030] nova-conductor[53893]: DEBUG nova.quota [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Getting quotas for project c18723dc952a4233846634205ce5bc23. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 757.490328] nova-conductor[53894]: DEBUG nova.quota [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Getting quotas for user cdda174a6ea54d4984293438287d97a9 and project 0ac54b53da9b4c9d99b34169cc521682. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 757.490410] nova-conductor[53893]: DEBUG nova.quota [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Getting quotas for user 0e5924ff92d443cd9206bdfd82447903 and project c18723dc952a4233846634205ce5bc23. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 757.499289] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] [instance: f5f4088e-0d79-4d6f-ae45-6ab2e5122e0a] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 757.500316] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.501211] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] [instance: 27788608-fef5-4163-932d-be6e2f60a541] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 757.501413] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 757.501459] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.501506] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 757.501617] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 757.501819] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 757.507771] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] [instance: f5f4088e-0d79-4d6f-ae45-6ab2e5122e0a] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 757.508733] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] [instance: 27788608-fef5-4163-932d-be6e2f60a541] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 757.509288] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.509288] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 757.509474] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 757.509743] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.511065] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 757.511065] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 757.536280] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.536520] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 757.536767] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.536957] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 757.537358] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 757.537573] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 757.706838] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Expecting reply to msg f8fbdf5c11704bbcb02a687345c14ac8 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 757.848701] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f8fbdf5c11704bbcb02a687345c14ac8 [ 757.848701] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Took 0.14 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 757.864022] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.864022] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 757.864022] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 757.906241] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.907040] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 757.907246] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 757.907646] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.907834] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 757.908164] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 757.920171] nova-conductor[53894]: DEBUG nova.quota [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Getting quotas for project fae9b11612ad41c39f02b336b91bfb37. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 757.930137] nova-conductor[53894]: DEBUG nova.quota [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Getting quotas for user 993b0efa2a084cd7936b7aff0fd5f2fd and project fae9b11612ad41c39f02b336b91bfb37. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 757.944734] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] [instance: 07d2f1e7-c08e-434c-aea7-941ef75f16ba] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 757.945296] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.945512] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 757.945952] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 757.958090] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] [instance: 07d2f1e7-c08e-434c-aea7-941ef75f16ba] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 757.958090] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.958090] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 757.958090] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 757.989465] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 757.989830] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 757.990105] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 758.250224] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] [instance: 807b279c-5934-4b4b-977d-c02a8dcbbdb6] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 8d3bba46-c3ed-46dc-b832-da5d946bb2f5, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 807b279c-5934-4b4b-977d-c02a8dcbbdb6 was re-scheduled: Binding failed for port 8d3bba46-c3ed-46dc-b832-da5d946bb2f5, please check neutron logs for more information.\n'] [ 758.253020] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 758.253020] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 807b279c-5934-4b4b-977d-c02a8dcbbdb6.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 807b279c-5934-4b4b-977d-c02a8dcbbdb6. [ 758.253020] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] [instance: 807b279c-5934-4b4b-977d-c02a8dcbbdb6] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 807b279c-5934-4b4b-977d-c02a8dcbbdb6. [ 758.291573] nova-conductor[53893]: DEBUG nova.network.neutron [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] [instance: 807b279c-5934-4b4b-977d-c02a8dcbbdb6] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 758.463148] nova-conductor[53893]: DEBUG nova.network.neutron [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] [instance: 807b279c-5934-4b4b-977d-c02a8dcbbdb6] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 758.479349] nova-conductor[53893]: DEBUG nova.network.neutron [None req-9d612945-e56a-4220-bed8-9dc166b58f9b tempest-ServerDiagnosticsTest-448951502 tempest-ServerDiagnosticsTest-448951502-project-member] [instance: 807b279c-5934-4b4b-977d-c02a8dcbbdb6] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 759.073400] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 51ae3e83-1cf3-4f56-b48d-b436ae84d706] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 010a6d62-e081-47f2-83de-049e88757d9d, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 51ae3e83-1cf3-4f56-b48d-b436ae84d706 was re-scheduled: Binding failed for port 010a6d62-e081-47f2-83de-049e88757d9d, please check neutron logs for more information.\n'] [ 759.074182] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 759.074758] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 51ae3e83-1cf3-4f56-b48d-b436ae84d706.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 51ae3e83-1cf3-4f56-b48d-b436ae84d706. [ 759.074758] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 51ae3e83-1cf3-4f56-b48d-b436ae84d706] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 51ae3e83-1cf3-4f56-b48d-b436ae84d706. [ 759.108322] nova-conductor[53894]: DEBUG nova.network.neutron [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 51ae3e83-1cf3-4f56-b48d-b436ae84d706] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 759.206069] nova-conductor[53894]: DEBUG nova.network.neutron [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 51ae3e83-1cf3-4f56-b48d-b436ae84d706] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 759.214376] nova-conductor[53894]: DEBUG nova.network.neutron [None req-eaa2de39-c1c4-4100-903f-cc88de1d1446 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 51ae3e83-1cf3-4f56-b48d-b436ae84d706] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 760.489870] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] [instance: 681fb733-cd37-4f73-a487-e4856206907f] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port cf2a5b97-befe-4276-9dc3-bea4d29d0e6a, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 681fb733-cd37-4f73-a487-e4856206907f was re-scheduled: Binding failed for port cf2a5b97-befe-4276-9dc3-bea4d29d0e6a, please check neutron logs for more information.\n'] [ 760.491655] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 760.491655] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 681fb733-cd37-4f73-a487-e4856206907f.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 681fb733-cd37-4f73-a487-e4856206907f. [ 760.491655] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] [instance: 681fb733-cd37-4f73-a487-e4856206907f] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 681fb733-cd37-4f73-a487-e4856206907f. [ 760.527669] nova-conductor[53894]: DEBUG nova.network.neutron [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] [instance: 681fb733-cd37-4f73-a487-e4856206907f] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 760.602912] nova-conductor[53894]: DEBUG nova.network.neutron [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] [instance: 681fb733-cd37-4f73-a487-e4856206907f] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 760.607027] nova-conductor[53894]: DEBUG nova.network.neutron [None req-66694e96-b294-4468-97c5-98a281230e3a tempest-ImagesOneServerTestJSON-327664204 tempest-ImagesOneServerTestJSON-327664204-project-member] [instance: 681fb733-cd37-4f73-a487-e4856206907f] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 761.096761] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Expecting reply to msg e4482c0ec1534707accc4f2478361666 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 761.254734] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e4482c0ec1534707accc4f2478361666 [ 761.255626] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Took 0.16 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 761.279855] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 761.280170] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 761.280551] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 761.313388] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 761.313611] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 761.313785] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 761.314188] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 761.314382] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 761.314587] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 761.323808] nova-conductor[53893]: DEBUG nova.quota [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Getting quotas for project f0bb8755136749c6a5166a19e20059df. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 761.326306] nova-conductor[53893]: DEBUG nova.quota [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Getting quotas for user f77e9c81e9eb4097bd16d9e763423d3d and project f0bb8755136749c6a5166a19e20059df. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 761.332073] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] [instance: 791543a0-ad32-4b51-9d21-b5cc72e480ff] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 761.332590] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 761.332746] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 761.332853] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 761.337812] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] [instance: 791543a0-ad32-4b51-9d21-b5cc72e480ff] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 761.338539] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 761.338757] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 761.338928] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 761.381963] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 761.382246] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 761.382433] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 761.483377] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] [instance: d16dff71-2dab-469a-8cb1-40ed086c42d1] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 8dfc2970-8f62-41a9-be95-d63bee3b3b33, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance d16dff71-2dab-469a-8cb1-40ed086c42d1 was re-scheduled: Binding failed for port 8dfc2970-8f62-41a9-be95-d63bee3b3b33, please check neutron logs for more information.\n'] [ 761.484106] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 761.484564] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d16dff71-2dab-469a-8cb1-40ed086c42d1.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d16dff71-2dab-469a-8cb1-40ed086c42d1. [ 761.487197] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] [instance: d16dff71-2dab-469a-8cb1-40ed086c42d1] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d16dff71-2dab-469a-8cb1-40ed086c42d1. [ 761.533725] nova-conductor[53893]: DEBUG nova.network.neutron [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] [instance: d16dff71-2dab-469a-8cb1-40ed086c42d1] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 761.625317] nova-conductor[53893]: DEBUG nova.network.neutron [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] [instance: d16dff71-2dab-469a-8cb1-40ed086c42d1] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 761.629460] nova-conductor[53893]: DEBUG nova.network.neutron [None req-33825971-5a59-4099-9f6b-088bd9c9c766 tempest-ServerAddressesNegativeTestJSON-1256841183 tempest-ServerAddressesNegativeTestJSON-1256841183-project-member] [instance: d16dff71-2dab-469a-8cb1-40ed086c42d1] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 761.872951] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 45a40160-c224-4a8f-8e92-26d770d4ff4f] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port e4f28593-3f76-493e-acd6-31f97463f969, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 45a40160-c224-4a8f-8e92-26d770d4ff4f was re-scheduled: Binding failed for port e4f28593-3f76-493e-acd6-31f97463f969, please check neutron logs for more information.\n'] [ 761.873458] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 761.873679] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 45a40160-c224-4a8f-8e92-26d770d4ff4f.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 45a40160-c224-4a8f-8e92-26d770d4ff4f. [ 761.873882] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 45a40160-c224-4a8f-8e92-26d770d4ff4f] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 45a40160-c224-4a8f-8e92-26d770d4ff4f. [ 761.902269] nova-conductor[53893]: DEBUG nova.network.neutron [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 45a40160-c224-4a8f-8e92-26d770d4ff4f] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 761.996609] nova-conductor[53893]: DEBUG nova.network.neutron [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 45a40160-c224-4a8f-8e92-26d770d4ff4f] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 762.000196] nova-conductor[53893]: DEBUG nova.network.neutron [None req-c4759a7c-e8c1-4013-b1a8-5676310751b1 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 45a40160-c224-4a8f-8e92-26d770d4ff4f] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 762.877322] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Expecting reply to msg 211cb0ea4ffc4292a42e0979988ab1dc in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 763.014298] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 211cb0ea4ffc4292a42e0979988ab1dc [ 763.015908] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Took 0.14 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 763.043771] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 763.043771] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 763.043771] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 763.085274] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 763.085506] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 763.085669] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 763.086167] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 763.086241] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 763.086601] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 763.096358] nova-conductor[53893]: DEBUG nova.quota [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Getting quotas for project 05666a0abf0b433184c14ff43e6e82c1. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 763.098987] nova-conductor[53893]: DEBUG nova.quota [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Getting quotas for user bfb9e86398ee44b7b9e06dcdc29f8ff6 and project 05666a0abf0b433184c14ff43e6e82c1. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 763.108022] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] [instance: 8b0ba692-56df-47d6-ad26-cbfb1c3bb00a] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 763.108325] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 763.108554] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 763.108705] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 763.121468] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] [instance: 8b0ba692-56df-47d6-ad26-cbfb1c3bb00a] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 763.122196] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 763.122963] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 763.123169] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 763.146149] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 763.146384] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 763.146557] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 764.662891] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] [instance: c588118f-5b15-4262-8d21-c4fbc3088f1f] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port cfc56a70-5dc7-481d-ab71-f33ec5765463, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance c588118f-5b15-4262-8d21-c4fbc3088f1f was re-scheduled: Binding failed for port cfc56a70-5dc7-481d-ab71-f33ec5765463, please check neutron logs for more information.\n'] [ 764.664076] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 764.666127] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance c588118f-5b15-4262-8d21-c4fbc3088f1f.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance c588118f-5b15-4262-8d21-c4fbc3088f1f. [ 764.666429] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] [instance: c588118f-5b15-4262-8d21-c4fbc3088f1f] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance c588118f-5b15-4262-8d21-c4fbc3088f1f. [ 764.732193] nova-conductor[53894]: DEBUG nova.network.neutron [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] [instance: c588118f-5b15-4262-8d21-c4fbc3088f1f] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 764.946478] nova-conductor[53894]: DEBUG nova.network.neutron [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] [instance: c588118f-5b15-4262-8d21-c4fbc3088f1f] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 764.951381] nova-conductor[53894]: DEBUG nova.network.neutron [None req-36578c38-88dc-48c7-bda3-cdcf04efea93 tempest-ServerMetadataTestJSON-1448747604 tempest-ServerMetadataTestJSON-1448747604-project-member] [instance: c588118f-5b15-4262-8d21-c4fbc3088f1f] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 765.725919] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] [instance: de519927-4dbd-4e6b-ba0a-d02d01950749] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port ff3cb204-5637-44ad-955a-535d5ce72a74, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance de519927-4dbd-4e6b-ba0a-d02d01950749 was re-scheduled: Binding failed for port ff3cb204-5637-44ad-955a-535d5ce72a74, please check neutron logs for more information.\n'] [ 765.726628] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 765.726872] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance de519927-4dbd-4e6b-ba0a-d02d01950749.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance de519927-4dbd-4e6b-ba0a-d02d01950749. [ 765.727206] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] [instance: de519927-4dbd-4e6b-ba0a-d02d01950749] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance de519927-4dbd-4e6b-ba0a-d02d01950749. [ 765.753664] nova-conductor[53893]: DEBUG nova.network.neutron [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] [instance: de519927-4dbd-4e6b-ba0a-d02d01950749] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 765.820134] nova-conductor[53893]: DEBUG nova.network.neutron [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] [instance: de519927-4dbd-4e6b-ba0a-d02d01950749] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 765.823813] nova-conductor[53893]: DEBUG nova.network.neutron [None req-59db6685-0dbe-4790-98e8-2893a7583856 tempest-AttachVolumeShelveTestJSON-2013570722 tempest-AttachVolumeShelveTestJSON-2013570722-project-member] [instance: de519927-4dbd-4e6b-ba0a-d02d01950749] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 767.030771] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 3d6ead4a-228d-48da-89b3-4fa99f6299d2] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port dd2bb230-848c-4e9a-8343-59436d2fbda8, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 3d6ead4a-228d-48da-89b3-4fa99f6299d2 was re-scheduled: Binding failed for port dd2bb230-848c-4e9a-8343-59436d2fbda8, please check neutron logs for more information.\n'] [ 767.030771] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 767.030771] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 3d6ead4a-228d-48da-89b3-4fa99f6299d2.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 3d6ead4a-228d-48da-89b3-4fa99f6299d2. [ 767.032442] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 3d6ead4a-228d-48da-89b3-4fa99f6299d2] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 3d6ead4a-228d-48da-89b3-4fa99f6299d2. [ 767.059996] nova-conductor[53894]: DEBUG nova.network.neutron [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 3d6ead4a-228d-48da-89b3-4fa99f6299d2] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 767.108350] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Expecting reply to msg a1069701742d4e09a2f314a6f54ffb5d in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 767.109897] nova-conductor[53894]: DEBUG nova.network.neutron [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 3d6ead4a-228d-48da-89b3-4fa99f6299d2] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 767.115703] nova-conductor[53894]: DEBUG nova.network.neutron [None req-d1f65603-0837-4a83-bf94-b8a91ee56633 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 3d6ead4a-228d-48da-89b3-4fa99f6299d2] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 767.273809] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a1069701742d4e09a2f314a6f54ffb5d [ 767.274075] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Took 0.17 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 767.298295] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 767.298755] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 767.301030] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 767.332140] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Expecting reply to msg 4792eb420883438a9e55b980bbe287d1 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 767.378975] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 767.378975] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 767.379587] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 767.380051] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 767.380936] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 767.380936] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 767.397993] nova-conductor[53894]: DEBUG nova.quota [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Getting quotas for project 9dbc2efe50ed4ae5a5f0cf6f492a20bf. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 767.402071] nova-conductor[53894]: DEBUG nova.quota [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Getting quotas for user 7ce2159a0ccb46c89a7574d04142e926 and project 9dbc2efe50ed4ae5a5f0cf6f492a20bf. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 767.412023] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: b6ee7d2a-dec8-4dad-b220-483e3313da31] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 767.412023] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 767.412023] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 767.412023] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 767.420741] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: b6ee7d2a-dec8-4dad-b220-483e3313da31] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 767.421892] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 767.422357] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 767.422721] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 767.449754] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 767.449973] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 767.450158] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 767.549563] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4792eb420883438a9e55b980bbe287d1 [ 767.550457] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Took 0.22 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 767.579187] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 767.579570] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 767.579745] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 767.631920] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 767.632132] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 767.632303] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 767.632672] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 767.632964] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 767.633065] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 767.648393] nova-conductor[53894]: DEBUG nova.quota [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Getting quotas for project f0bb8755136749c6a5166a19e20059df. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 767.652957] nova-conductor[53894]: DEBUG nova.quota [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Getting quotas for user f77e9c81e9eb4097bd16d9e763423d3d and project f0bb8755136749c6a5166a19e20059df. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 767.659757] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] [instance: 9edf2808-0bc5-4951-9d0c-7806a4ec1b9f] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 767.660289] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 767.660495] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 767.660663] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 767.665150] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] [instance: 9edf2808-0bc5-4951-9d0c-7806a4ec1b9f] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 767.665876] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 767.666055] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 767.666231] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 767.690379] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 767.690624] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 767.690794] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 768.301916] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] [instance: f5f4088e-0d79-4d6f-ae45-6ab2e5122e0a] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 65cc7798-0ef5-4fd5-8396-eed3859fc1af, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance f5f4088e-0d79-4d6f-ae45-6ab2e5122e0a was re-scheduled: Binding failed for port 65cc7798-0ef5-4fd5-8396-eed3859fc1af, please check neutron logs for more information.\n'] [ 768.301916] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 768.302234] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance f5f4088e-0d79-4d6f-ae45-6ab2e5122e0a.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance f5f4088e-0d79-4d6f-ae45-6ab2e5122e0a. [ 768.302422] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] [instance: f5f4088e-0d79-4d6f-ae45-6ab2e5122e0a] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance f5f4088e-0d79-4d6f-ae45-6ab2e5122e0a. [ 768.337977] nova-conductor[53893]: DEBUG nova.network.neutron [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] [instance: f5f4088e-0d79-4d6f-ae45-6ab2e5122e0a] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 768.384503] nova-conductor[53893]: DEBUG nova.network.neutron [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] [instance: f5f4088e-0d79-4d6f-ae45-6ab2e5122e0a] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 768.390253] nova-conductor[53893]: DEBUG nova.network.neutron [None req-c70b2b64-1ac4-43d4-8b2d-ab4e1e814afe tempest-FloatingIPsAssociationNegativeTestJSON-1743565394 tempest-FloatingIPsAssociationNegativeTestJSON-1743565394-project-member] [instance: f5f4088e-0d79-4d6f-ae45-6ab2e5122e0a] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 768.543887] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] [instance: 27788608-fef5-4163-932d-be6e2f60a541] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 68b29d7a-88f7-4130-a084-890a355e3dbb, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 27788608-fef5-4163-932d-be6e2f60a541 was re-scheduled: Binding failed for port 68b29d7a-88f7-4130-a084-890a355e3dbb, please check neutron logs for more information.\n'] [ 768.545222] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 768.545585] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 27788608-fef5-4163-932d-be6e2f60a541.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 27788608-fef5-4163-932d-be6e2f60a541. [ 768.545811] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] [instance: 27788608-fef5-4163-932d-be6e2f60a541] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 27788608-fef5-4163-932d-be6e2f60a541. [ 768.569364] nova-conductor[53893]: DEBUG nova.network.neutron [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] [instance: 27788608-fef5-4163-932d-be6e2f60a541] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 768.613879] nova-conductor[53893]: DEBUG nova.network.neutron [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] [instance: 27788608-fef5-4163-932d-be6e2f60a541] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 768.619776] nova-conductor[53893]: DEBUG nova.network.neutron [None req-3c4169f2-d315-4bc5-868d-348478a0e99b tempest-InstanceActionsV221TestJSON-559212044 tempest-InstanceActionsV221TestJSON-559212044-project-member] [instance: 27788608-fef5-4163-932d-be6e2f60a541] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 768.693059] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Expecting reply to msg 93525d12ee8f4308a2c31165827fa454 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 768.908858] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 93525d12ee8f4308a2c31165827fa454 [ 768.908858] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Took 0.22 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 768.935328] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 768.937365] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 768.937365] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 769.010334] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Expecting reply to msg 2286ea26ce7443f2afc471953eb1ab61 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 769.025112] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 769.025112] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 769.025112] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 769.026456] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 769.026527] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.002s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 769.027415] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 769.043688] nova-conductor[53894]: DEBUG nova.quota [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Getting quotas for project 0543ecd93ed4491286c3c99f7d567ed4. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 769.046063] nova-conductor[53894]: DEBUG nova.quota [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Getting quotas for user c80a54c84c9445e5b0abb5ca5eedc414 and project 0543ecd93ed4491286c3c99f7d567ed4. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 769.053194] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] [instance: 017f274d-c305-4aff-977a-c8bb2827880f] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 769.053716] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 769.053926] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 769.054110] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 769.060688] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] [instance: 017f274d-c305-4aff-977a-c8bb2827880f] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 769.061327] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 769.061534] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 769.061703] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 769.082588] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 769.082670] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 769.084962] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 769.205173] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 2286ea26ce7443f2afc471953eb1ab61 [ 769.205990] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Took 0.20 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 769.226314] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 769.226680] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 769.226905] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 769.276864] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 769.277244] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 769.277404] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 769.279477] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 769.279477] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 769.279477] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 769.295641] nova-conductor[53893]: DEBUG nova.quota [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Getting quotas for project afc5e909ec5c4dd983ece5aa3236910f. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 769.303467] nova-conductor[53893]: DEBUG nova.quota [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Getting quotas for user ee5a21ff43314c1a857f6958056173f9 and project afc5e909ec5c4dd983ece5aa3236910f. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 769.313549] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 6905ca7f-445e-45f8-8558-b119560a4216] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 769.314007] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 769.314232] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 769.314633] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 769.318698] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 6905ca7f-445e-45f8-8558-b119560a4216] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 769.319364] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 769.319570] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 769.319740] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 769.339595] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 769.339830] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 769.340009] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 770.125548] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] [instance: 791543a0-ad32-4b51-9d21-b5cc72e480ff] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port a5a3c5a8-89c7-42ba-9904-f5dd12e4c9f3, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 791543a0-ad32-4b51-9d21-b5cc72e480ff was re-scheduled: Binding failed for port a5a3c5a8-89c7-42ba-9904-f5dd12e4c9f3, please check neutron logs for more information.\n'] [ 770.126666] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 770.126895] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 791543a0-ad32-4b51-9d21-b5cc72e480ff.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 791543a0-ad32-4b51-9d21-b5cc72e480ff. [ 770.127093] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] [instance: 791543a0-ad32-4b51-9d21-b5cc72e480ff] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 791543a0-ad32-4b51-9d21-b5cc72e480ff. [ 770.163050] nova-conductor[53893]: DEBUG nova.network.neutron [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] [instance: 791543a0-ad32-4b51-9d21-b5cc72e480ff] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 770.224786] nova-conductor[53893]: DEBUG nova.network.neutron [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] [instance: 791543a0-ad32-4b51-9d21-b5cc72e480ff] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 770.229044] nova-conductor[53893]: DEBUG nova.network.neutron [None req-626ffeab-4788-4a12-a502-6377da7152c2 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] [instance: 791543a0-ad32-4b51-9d21-b5cc72e480ff] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 771.471140] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] [instance: 8b0ba692-56df-47d6-ad26-cbfb1c3bb00a] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 7e591090-397c-446d-bc6c-60ba035cbddd, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 8b0ba692-56df-47d6-ad26-cbfb1c3bb00a was re-scheduled: Binding failed for port 7e591090-397c-446d-bc6c-60ba035cbddd, please check neutron logs for more information.\n'] [ 771.474171] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 771.474806] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8b0ba692-56df-47d6-ad26-cbfb1c3bb00a.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8b0ba692-56df-47d6-ad26-cbfb1c3bb00a. [ 771.478919] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] [instance: 8b0ba692-56df-47d6-ad26-cbfb1c3bb00a] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8b0ba692-56df-47d6-ad26-cbfb1c3bb00a. [ 771.530498] nova-conductor[53893]: DEBUG nova.network.neutron [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] [instance: 8b0ba692-56df-47d6-ad26-cbfb1c3bb00a] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 771.622874] nova-conductor[53893]: DEBUG nova.network.neutron [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] [instance: 8b0ba692-56df-47d6-ad26-cbfb1c3bb00a] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 771.634484] nova-conductor[53893]: DEBUG nova.network.neutron [None req-65aa97a9-3521-4ecd-b217-53071e696335 tempest-MigrationsAdminTest-1912133545 tempest-MigrationsAdminTest-1912133545-project-member] [instance: 8b0ba692-56df-47d6-ad26-cbfb1c3bb00a] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 773.019958] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Expecting reply to msg 7fb422004ecb4e589e58da6b293eac0e in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 773.151691] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7fb422004ecb4e589e58da6b293eac0e [ 773.152455] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Took 0.14 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 773.169164] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 773.169461] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 773.169719] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 773.209235] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 773.209446] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 773.209620] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 773.209986] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 773.210209] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 773.210375] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 773.219488] nova-conductor[53893]: DEBUG nova.quota [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Getting quotas for project bff3be1976444e58a2b7be93d47f50ef. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 773.222016] nova-conductor[53893]: DEBUG nova.quota [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Getting quotas for user 84f05a61e0f6499bb35c44d254226249 and project bff3be1976444e58a2b7be93d47f50ef. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 773.228335] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 3b261d28-a9b1-4551-8e39-2108e825aedc] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 773.228684] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 773.228934] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 773.229131] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 773.235219] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 3b261d28-a9b1-4551-8e39-2108e825aedc] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 773.236083] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 773.236295] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 773.236464] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 773.256706] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 773.256930] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 773.257115] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 776.823999] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Expecting reply to msg 04e0acde2b544004a6a8cc64e8d55cc7 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 777.018239] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 04e0acde2b544004a6a8cc64e8d55cc7 [ 777.019501] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Took 0.20 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 777.039797] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 777.040281] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 777.040570] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 777.091689] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 777.091926] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 777.092117] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 777.092489] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 777.092670] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 777.092827] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 777.108893] nova-conductor[53894]: DEBUG nova.quota [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Getting quotas for project 3dfb77f12805418eaa6127fc75becec0. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 777.116517] nova-conductor[53894]: DEBUG nova.quota [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Getting quotas for user 789177a2f7be455cadec45cf03d67521 and project 3dfb77f12805418eaa6127fc75becec0. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 777.123284] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 18087c89-5fb4-4a16-a3bb-d48712ce25f5] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 777.123795] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 777.124015] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 777.124357] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 777.130079] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 18087c89-5fb4-4a16-a3bb-d48712ce25f5] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 777.130923] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 777.131143] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 777.131377] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 777.151376] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 777.151876] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 777.151876] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 778.673644] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Expecting reply to msg e830ba1af91c481191710c9bbebdb65b in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 778.830393] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e830ba1af91c481191710c9bbebdb65b [ 778.831273] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Took 0.16 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 778.848148] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 778.848397] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 778.848568] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 778.921029] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 778.921029] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 778.921029] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 778.921029] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 778.921029] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 778.921029] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 778.942782] nova-conductor[53893]: DEBUG nova.quota [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Getting quotas for project 57787a709d744ea4a19a2cfb923d89d4. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 778.946013] nova-conductor[53893]: DEBUG nova.quota [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Getting quotas for user 2d6b88b1f4394a63b32be00fc9dcc761 and project 57787a709d744ea4a19a2cfb923d89d4. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 778.952767] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: 1eebe96f-c03f-4069-99d0-ea2a50de6f35] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 778.952946] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 778.953157] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 778.953333] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 778.960764] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: 1eebe96f-c03f-4069-99d0-ea2a50de6f35] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 778.961472] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 778.961674] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 778.961840] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 778.984182] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 778.984497] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 778.984702] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 780.327667] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] [instance: 9edf2808-0bc5-4951-9d0c-7806a4ec1b9f] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port d5333c50-e5a1-43e0-9f2b-1e477c0113f7, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 9edf2808-0bc5-4951-9d0c-7806a4ec1b9f was re-scheduled: Binding failed for port d5333c50-e5a1-43e0-9f2b-1e477c0113f7, please check neutron logs for more information.\n'] [ 780.329833] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 780.330112] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 9edf2808-0bc5-4951-9d0c-7806a4ec1b9f.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 9edf2808-0bc5-4951-9d0c-7806a4ec1b9f. [ 780.330506] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] [instance: 9edf2808-0bc5-4951-9d0c-7806a4ec1b9f] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 9edf2808-0bc5-4951-9d0c-7806a4ec1b9f. [ 780.436586] nova-conductor[53894]: DEBUG nova.network.neutron [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] [instance: 9edf2808-0bc5-4951-9d0c-7806a4ec1b9f] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 780.665909] nova-conductor[53894]: DEBUG nova.network.neutron [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] [instance: 9edf2808-0bc5-4951-9d0c-7806a4ec1b9f] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 780.677982] nova-conductor[53894]: DEBUG nova.network.neutron [None req-126df6f1-5b4e-487d-b274-cbfae3442d95 tempest-ServerRescueNegativeTestJSON-1171749554 tempest-ServerRescueNegativeTestJSON-1171749554-project-member] [instance: 9edf2808-0bc5-4951-9d0c-7806a4ec1b9f] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 781.552548] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: b6ee7d2a-dec8-4dad-b220-483e3313da31] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 35fb7071-e5a9-4272-ae77-a82b6638f4d7, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance b6ee7d2a-dec8-4dad-b220-483e3313da31 was re-scheduled: Binding failed for port 35fb7071-e5a9-4272-ae77-a82b6638f4d7, please check neutron logs for more information.\n'] [ 781.553394] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 781.553546] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b6ee7d2a-dec8-4dad-b220-483e3313da31.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b6ee7d2a-dec8-4dad-b220-483e3313da31. [ 781.554091] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: b6ee7d2a-dec8-4dad-b220-483e3313da31] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b6ee7d2a-dec8-4dad-b220-483e3313da31. [ 781.627493] nova-conductor[53894]: DEBUG nova.network.neutron [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: b6ee7d2a-dec8-4dad-b220-483e3313da31] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 781.908960] nova-conductor[53894]: DEBUG nova.network.neutron [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: b6ee7d2a-dec8-4dad-b220-483e3313da31] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 781.914377] nova-conductor[53894]: DEBUG nova.network.neutron [None req-093ecb04-435b-4dfd-bce5-0d61b635343c tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: b6ee7d2a-dec8-4dad-b220-483e3313da31] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 782.120101] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] [instance: 017f274d-c305-4aff-977a-c8bb2827880f] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port a050d813-7f37-4cc8-bb22-d6596b4f310a, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 017f274d-c305-4aff-977a-c8bb2827880f was re-scheduled: Binding failed for port a050d813-7f37-4cc8-bb22-d6596b4f310a, please check neutron logs for more information.\n'] [ 782.120101] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 782.120101] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 017f274d-c305-4aff-977a-c8bb2827880f.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 017f274d-c305-4aff-977a-c8bb2827880f. [ 782.120101] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] [instance: 017f274d-c305-4aff-977a-c8bb2827880f] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 017f274d-c305-4aff-977a-c8bb2827880f. [ 782.145132] nova-conductor[53893]: DEBUG nova.network.neutron [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] [instance: 017f274d-c305-4aff-977a-c8bb2827880f] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 782.267415] nova-conductor[53893]: DEBUG nova.network.neutron [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] [instance: 017f274d-c305-4aff-977a-c8bb2827880f] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 782.274017] nova-conductor[53893]: DEBUG nova.network.neutron [None req-bf15dfa0-9f6b-413d-acac-daef7b069f4f tempest-AttachInterfacesV270Test-749677004 tempest-AttachInterfacesV270Test-749677004-project-member] [instance: 017f274d-c305-4aff-977a-c8bb2827880f] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 782.738813] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 6905ca7f-445e-45f8-8558-b119560a4216] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port ef14754b-4ab7-4ab3-b64d-f4166bee7327, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 6905ca7f-445e-45f8-8558-b119560a4216 was re-scheduled: Binding failed for port ef14754b-4ab7-4ab3-b64d-f4166bee7327, please check neutron logs for more information.\n'] [ 782.742025] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 782.742025] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 6905ca7f-445e-45f8-8558-b119560a4216.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 6905ca7f-445e-45f8-8558-b119560a4216. [ 782.742025] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 6905ca7f-445e-45f8-8558-b119560a4216] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 6905ca7f-445e-45f8-8558-b119560a4216. [ 782.765943] nova-conductor[53894]: DEBUG nova.network.neutron [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 6905ca7f-445e-45f8-8558-b119560a4216] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 782.849113] nova-conductor[53894]: DEBUG nova.network.neutron [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 6905ca7f-445e-45f8-8558-b119560a4216] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 782.851270] nova-conductor[53894]: DEBUG nova.network.neutron [None req-8bbbdfac-e3d1-4b30-a07e-ae5e9dad2824 tempest-ServerDiskConfigTestJSON-779666606 tempest-ServerDiskConfigTestJSON-779666606-project-member] [instance: 6905ca7f-445e-45f8-8558-b119560a4216] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 786.461719] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:UPDATE=367,nova_cell1:SELECT=1255,nova_cell1:SAVEPOINT=71,nova_cell1:RELEASE=71,nova_cell1:INSERT=48,nova_cell1:DELETE=2 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 786.797469] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Expecting reply to msg 7465e7a23aa24aa0ba61be3b519ec557 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 787.021932] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7465e7a23aa24aa0ba61be3b519ec557 [ 787.022758] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Took 0.23 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 787.045606] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 787.045606] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 787.046161] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 787.097565] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell0:SELECT=24 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 787.099169] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 787.099169] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 787.100763] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 787.100763] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 787.100763] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 787.100763] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 787.115594] nova-conductor[53894]: DEBUG nova.quota [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Getting quotas for project 2a98ff03844e4dbf912f40034f907789. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 787.122254] nova-conductor[53894]: DEBUG nova.quota [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Getting quotas for user a9a8a6f1cd534d9c807e7581aa019685 and project 2a98ff03844e4dbf912f40034f907789. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 787.134562] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] [instance: 28bcec42-4fb0-4ef1-b882-6224fdbcec16] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 787.135463] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 787.135620] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 787.135918] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 787.143068] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] [instance: 28bcec42-4fb0-4ef1-b882-6224fdbcec16] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 787.144406] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 787.144734] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 787.145121] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 787.173077] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 787.173366] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 787.173542] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 788.086080] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 3b261d28-a9b1-4551-8e39-2108e825aedc] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 120b7e4f-ee57-4bf0-a4a9-ecf2ab820872, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 3b261d28-a9b1-4551-8e39-2108e825aedc was re-scheduled: Binding failed for port 120b7e4f-ee57-4bf0-a4a9-ecf2ab820872, please check neutron logs for more information.\n'] [ 788.087378] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 788.087882] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 3b261d28-a9b1-4551-8e39-2108e825aedc.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 3b261d28-a9b1-4551-8e39-2108e825aedc. [ 788.088090] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 3b261d28-a9b1-4551-8e39-2108e825aedc] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 3b261d28-a9b1-4551-8e39-2108e825aedc. [ 788.122324] nova-conductor[53894]: DEBUG nova.network.neutron [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 3b261d28-a9b1-4551-8e39-2108e825aedc] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 788.242484] nova-conductor[53894]: DEBUG nova.network.neutron [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 3b261d28-a9b1-4551-8e39-2108e825aedc] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 788.245998] nova-conductor[53894]: DEBUG nova.network.neutron [None req-7a53fd6b-ce3a-4a6f-bbc3-6c1916918b1a tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 3b261d28-a9b1-4551-8e39-2108e825aedc] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 788.926054] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell0:SELECT=15 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 788.965683] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=71,nova_cell1:SAVEPOINT=4,nova_cell1:INSERT=156,nova_cell1:RELEASE=4 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 788.975749] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=92,nova_api:UPDATE=4,nova_api:DELETE=5 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 788.979078] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=96,nova_api:UPDATE=4,nova_api:DELETE=3 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 790.616749] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Expecting reply to msg 914b4440cbb04b56b4f9cb207a455df4 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 790.765410] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 914b4440cbb04b56b4f9cb207a455df4 [ 790.766976] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Took 0.15 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 790.790553] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 790.790848] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 790.791369] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 790.854746] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 790.854746] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 790.854746] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 790.855161] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 790.855352] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 790.856303] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 790.858978] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=123,nova_cell1:INSERT=271,nova_cell1:SAVEPOINT=7,nova_cell1:RELEASE=7 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 790.868986] nova-conductor[53894]: DEBUG nova.quota [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Getting quotas for project b54f5acf42c64133afbb208929492c31. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 790.872112] nova-conductor[53894]: DEBUG nova.quota [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Getting quotas for user 06291742abbe4dee8092fba657b8ab91 and project b54f5acf42c64133afbb208929492c31. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 790.872814] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_api:SELECT=172,nova_api:DELETE=6,nova_api:UPDATE=3 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 790.873699] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_api:SELECT=156,nova_api:UPDATE=10,nova_api:DELETE=7 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 790.881812] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 11ca01d2-83e3-42c4-bef5-87459148e858] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 790.882298] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 790.882511] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 790.882674] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 790.888746] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 11ca01d2-83e3-42c4-bef5-87459148e858] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 790.889090] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 790.891030] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 790.891030] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 790.915230] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 790.915650] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 790.915873] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 791.944482] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Expecting reply to msg 03b6c91539ef4a48b1fb37d988ff3826 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 792.104425] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 03b6c91539ef4a48b1fb37d988ff3826 [ 792.105314] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Took 0.16 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 792.126925] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 792.126925] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 792.126925] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 792.162565] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:UPDATE=355,nova_cell1:SELECT=1231,nova_cell1:INSERT=56,nova_cell1:SAVEPOINT=60,nova_cell1:RELEASE=60 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 792.164771] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 792.165053] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 792.165263] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 792.166070] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 792.166324] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 792.166642] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 792.180161] nova-conductor[53894]: DEBUG nova.quota [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Getting quotas for project 9dbc2efe50ed4ae5a5f0cf6f492a20bf. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 792.182734] nova-conductor[53894]: DEBUG nova.quota [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Getting quotas for user 7ce2159a0ccb46c89a7574d04142e926 and project 9dbc2efe50ed4ae5a5f0cf6f492a20bf. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 792.189268] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: 72c150cd-b1f2-451d-bb6b-6b8b668e97a1] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 792.189810] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 792.190255] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 792.193519] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 792.199217] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: 72c150cd-b1f2-451d-bb6b-6b8b668e97a1] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 792.200603] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 792.200858] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 792.201040] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 792.219561] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 792.219772] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 792.219961] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 792.878785] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Expecting reply to msg 7d23fdb1ea804b8fb62e601cf1e13884 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 793.097426] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7d23fdb1ea804b8fb62e601cf1e13884 [ 793.098362] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Took 0.22 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 793.133281] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 793.133502] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 793.133692] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 793.197752] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 793.197984] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 793.198199] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 793.198580] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 793.198759] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 793.201567] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 793.219192] nova-conductor[53893]: DEBUG nova.quota [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Getting quotas for project 1a71ad350ba241b5a3f933887cb0e8a4. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 793.224402] nova-conductor[53893]: DEBUG nova.quota [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Getting quotas for user 084fb304b8aa400ebcd3bab4404c36ae and project 1a71ad350ba241b5a3f933887cb0e8a4. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 793.241615] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] [instance: af3ae6a1-44e4-49f7-905c-1480ec3ad0a9] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 793.242387] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 793.242645] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 793.244347] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 793.259251] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] [instance: af3ae6a1-44e4-49f7-905c-1480ec3ad0a9] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 793.259251] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 793.259251] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 793.259447] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 793.286971] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 793.287648] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 793.287648] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 794.441404] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 18087c89-5fb4-4a16-a3bb-d48712ce25f5] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port b4396922-ab16-4a9a-82f6-838138d4a038, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 18087c89-5fb4-4a16-a3bb-d48712ce25f5 was re-scheduled: Binding failed for port b4396922-ab16-4a9a-82f6-838138d4a038, please check neutron logs for more information.\n'] [ 794.442033] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 794.442281] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 18087c89-5fb4-4a16-a3bb-d48712ce25f5.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 18087c89-5fb4-4a16-a3bb-d48712ce25f5. [ 794.442617] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 18087c89-5fb4-4a16-a3bb-d48712ce25f5] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 18087c89-5fb4-4a16-a3bb-d48712ce25f5. [ 794.472680] nova-conductor[53893]: DEBUG nova.network.neutron [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 18087c89-5fb4-4a16-a3bb-d48712ce25f5] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 794.627975] nova-conductor[53893]: DEBUG nova.network.neutron [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 18087c89-5fb4-4a16-a3bb-d48712ce25f5] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 794.633352] nova-conductor[53893]: DEBUG nova.network.neutron [None req-02eb3e51-f0d0-4e59-b4c7-e2fd7f680940 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 18087c89-5fb4-4a16-a3bb-d48712ce25f5] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 795.998649] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Expecting reply to msg c728b8889e514990a5fc1d396def3e7b in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 796.169392] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c728b8889e514990a5fc1d396def3e7b [ 796.170331] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Took 0.17 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 796.184811] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 796.185115] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 796.185337] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 796.230322] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 796.230572] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 796.230740] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 796.231183] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 796.231432] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 796.231631] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 796.241472] nova-conductor[53894]: DEBUG nova.quota [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Getting quotas for project bc87957366af49c0b5fa7a2297203f53. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 796.245745] nova-conductor[53894]: DEBUG nova.quota [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Getting quotas for user 797bb578a1db4de9b026ae356342e3eb and project bc87957366af49c0b5fa7a2297203f53. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 796.252176] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] [instance: 48b27adc-0bc7-44dd-9330-d0b30593f95f] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 796.252731] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 796.252945] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 796.253127] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 796.259562] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] [instance: 48b27adc-0bc7-44dd-9330-d0b30593f95f] block_device_mapping [BlockDeviceMapping(attachment_id=d2d631a4-764e-4f94-83a8-866a248d6077,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='volume',tag=None,updated_at=,uuid=,volume_id='434f3adf-e546-451c-af96-c91009ff3d7e',volume_size=1,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 796.260398] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 796.260646] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 796.260861] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 796.278874] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 796.279111] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 796.279331] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 796.681465] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: 1eebe96f-c03f-4069-99d0-ea2a50de6f35] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port bd590870-c0eb-4002-80ae-72ba3ded7e91, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 1eebe96f-c03f-4069-99d0-ea2a50de6f35 was re-scheduled: Binding failed for port bd590870-c0eb-4002-80ae-72ba3ded7e91, please check neutron logs for more information.\n'] [ 796.682439] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 796.682797] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1eebe96f-c03f-4069-99d0-ea2a50de6f35.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1eebe96f-c03f-4069-99d0-ea2a50de6f35. [ 796.683359] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: 1eebe96f-c03f-4069-99d0-ea2a50de6f35] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1eebe96f-c03f-4069-99d0-ea2a50de6f35. [ 796.716821] nova-conductor[53894]: DEBUG nova.network.neutron [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: 1eebe96f-c03f-4069-99d0-ea2a50de6f35] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 796.799210] nova-conductor[53894]: DEBUG nova.network.neutron [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: 1eebe96f-c03f-4069-99d0-ea2a50de6f35] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 796.807559] nova-conductor[53894]: DEBUG nova.network.neutron [None req-d361c577-3bb4-4524-abf8-56d6661ff30a tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: 1eebe96f-c03f-4069-99d0-ea2a50de6f35] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 797.009352] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Expecting reply to msg 5b316257451648059f107b9240a8a304 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 797.185670] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5b316257451648059f107b9240a8a304 [ 797.190239] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Took 0.18 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 797.211246] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 797.212066] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 797.212277] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 797.271837] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 797.272038] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 797.272196] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 797.272574] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 797.272760] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 797.272988] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 797.283783] nova-conductor[53893]: DEBUG nova.quota [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Getting quotas for project 92ff6ab8a7d84bd4a5d50103a3b14bc2. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 797.287770] nova-conductor[53893]: DEBUG nova.quota [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Getting quotas for user 769b9425a8e04075a3831fe07d41ccd6 and project 92ff6ab8a7d84bd4a5d50103a3b14bc2. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 797.300296] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] [instance: 93ecfc04-23c5-41fc-babf-1a5bd02769b8] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 797.300765] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 797.301316] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 797.301316] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 797.306860] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] [instance: 93ecfc04-23c5-41fc-babf-1a5bd02769b8] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 797.307547] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 797.307808] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 797.307917] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 797.330528] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 797.330528] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 797.331090] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 799.683317] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Expecting reply to msg 921a09bf8b854380b2b61f6ec912a06d in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 799.819903] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Expecting reply to msg 591daf2cdfcb403ab923919c910fad39 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 799.882797] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 921a09bf8b854380b2b61f6ec912a06d [ 799.883122] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Took 0.21 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 799.904744] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 799.904744] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 799.904744] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 799.961500] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 799.961731] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 799.961897] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 799.962285] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 799.962585] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 799.962669] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 799.973482] nova-conductor[53893]: DEBUG nova.quota [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Getting quotas for project bff3be1976444e58a2b7be93d47f50ef. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 799.977416] nova-conductor[53893]: DEBUG nova.quota [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Getting quotas for user 84f05a61e0f6499bb35c44d254226249 and project bff3be1976444e58a2b7be93d47f50ef. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 799.983637] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 7cf10fb8-e3db-44fb-b9c3-4b9bb92d18e3] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 799.984597] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 799.984597] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 799.984597] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 799.992983] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 7cf10fb8-e3db-44fb-b9c3-4b9bb92d18e3] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 799.993702] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 799.993905] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 799.994140] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 800.017207] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 800.017505] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 800.017949] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 800.126752] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 591daf2cdfcb403ab923919c910fad39 [ 800.126752] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Took 0.31 seconds to select destinations for 2 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 800.156480] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 800.156925] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 800.157322] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 800.198750] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 800.198750] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 800.198750] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 800.247989] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 800.248339] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 800.248535] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 800.248925] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 800.249158] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 800.249330] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 800.263046] nova-conductor[53894]: DEBUG nova.quota [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Getting quotas for project 201633229817424689c8fe5557911ae9. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 800.270202] nova-conductor[53894]: DEBUG nova.quota [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Getting quotas for user 185a047edb6241d59feb8ad5d2bbc749 and project 201633229817424689c8fe5557911ae9. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 800.280041] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: e600280c-2414-420d-bc8d-6e3e7979fccf] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 800.280545] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 800.280744] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 800.280906] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 800.286486] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: e600280c-2414-420d-bc8d-6e3e7979fccf] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 800.286486] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 800.286946] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 800.286946] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 800.312924] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 800.313165] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 800.313335] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 800.322673] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: 4ad1a310-5786-4bb6-87ff-72069f692eff] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 800.323196] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 800.323408] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 800.323580] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 800.327639] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: 4ad1a310-5786-4bb6-87ff-72069f692eff] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 800.328442] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 800.328665] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 800.328836] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 800.351807] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 800.351807] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 800.351807] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 802.390069] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Expecting reply to msg 8e272498c7764f98a3f3023e4fe33c97 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 802.454530] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Expecting reply to msg fa9702edc856403eb830171ccfd5c417 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 802.578367] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 8e272498c7764f98a3f3023e4fe33c97 [ 802.579324] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Took 0.19 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 802.596717] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 802.596976] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 802.597184] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 802.630763] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fa9702edc856403eb830171ccfd5c417 [ 802.634039] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Took 0.18 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 802.649774] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 802.650601] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 802.650601] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 802.656018] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 802.656018] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 802.656018] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 802.656248] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 802.656248] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 802.656248] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 802.670444] nova-conductor[53894]: DEBUG nova.quota [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Getting quotas for project 54b079f5fde443dd9ac1e0f1666e46c2. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 802.678231] nova-conductor[53894]: DEBUG nova.quota [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Getting quotas for user 0ec87e86b5114bde83f2ab525f42f271 and project 54b079f5fde443dd9ac1e0f1666e46c2. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 802.685307] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] [instance: 500259b0-a57d-43ff-9c88-46190b6a3a10] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 802.685778] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 802.686128] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 802.686309] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 802.696835] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] [instance: 500259b0-a57d-43ff-9c88-46190b6a3a10] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 802.696835] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 802.697174] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 802.697228] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 802.704306] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 802.704306] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 802.704306] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 802.704306] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 802.704306] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 802.704306] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 802.717790] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 802.718463] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 802.718675] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 802.720705] nova-conductor[53893]: DEBUG nova.quota [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Getting quotas for project b7300699b4cb49de89d096e3a1ac1778. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 802.726796] nova-conductor[53893]: DEBUG nova.quota [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Getting quotas for user bcf95f76d0f54cbebde1301f62b44bcd and project b7300699b4cb49de89d096e3a1ac1778. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 802.740870] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] [instance: f7d0db3d-9247-409a-bf1e-4b53c2368ddc] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 802.741254] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 802.741612] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 802.741827] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 802.744683] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] [instance: f7d0db3d-9247-409a-bf1e-4b53c2368ddc] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 802.745358] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 802.745559] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 802.745722] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 802.762267] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 802.762267] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 802.762267] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 803.904647] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 11ca01d2-83e3-42c4-bef5-87459148e858] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port c8c36b11-f4c4-41e7-984e-dbd996ee47d9, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 11ca01d2-83e3-42c4-bef5-87459148e858 was re-scheduled: Binding failed for port c8c36b11-f4c4-41e7-984e-dbd996ee47d9, please check neutron logs for more information.\n'] [ 803.909832] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 803.909832] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 11ca01d2-83e3-42c4-bef5-87459148e858.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 11ca01d2-83e3-42c4-bef5-87459148e858. [ 803.912184] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 11ca01d2-83e3-42c4-bef5-87459148e858] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 11ca01d2-83e3-42c4-bef5-87459148e858. [ 803.974592] nova-conductor[53893]: DEBUG nova.network.neutron [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 11ca01d2-83e3-42c4-bef5-87459148e858] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 804.065173] nova-conductor[53893]: DEBUG nova.network.neutron [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 11ca01d2-83e3-42c4-bef5-87459148e858] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 804.075019] nova-conductor[53893]: DEBUG nova.network.neutron [None req-dbcc0dcb-51d6-4649-a979-d110209f148b tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 11ca01d2-83e3-42c4-bef5-87459148e858] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 804.972281] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: 72c150cd-b1f2-451d-bb6b-6b8b668e97a1] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 1e3c1995-8599-4229-aaad-23c894ad768e, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 72c150cd-b1f2-451d-bb6b-6b8b668e97a1 was re-scheduled: Binding failed for port 1e3c1995-8599-4229-aaad-23c894ad768e, please check neutron logs for more information.\n'] [ 804.972641] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 804.972641] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 72c150cd-b1f2-451d-bb6b-6b8b668e97a1.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 72c150cd-b1f2-451d-bb6b-6b8b668e97a1. [ 804.973382] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: 72c150cd-b1f2-451d-bb6b-6b8b668e97a1] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 72c150cd-b1f2-451d-bb6b-6b8b668e97a1. [ 805.031522] nova-conductor[53893]: DEBUG nova.network.neutron [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: 72c150cd-b1f2-451d-bb6b-6b8b668e97a1] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 805.127228] nova-conductor[53893]: DEBUG nova.network.neutron [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: 72c150cd-b1f2-451d-bb6b-6b8b668e97a1] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 805.131797] nova-conductor[53893]: DEBUG nova.network.neutron [None req-edefb713-b593-4bfd-8772-3c7cb2678348 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: 72c150cd-b1f2-451d-bb6b-6b8b668e97a1] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 806.531758] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] [instance: af3ae6a1-44e4-49f7-905c-1480ec3ad0a9] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 53214105-fc54-4a7d-ac37-7cb3c832497a, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance af3ae6a1-44e4-49f7-905c-1480ec3ad0a9 was re-scheduled: Binding failed for port 53214105-fc54-4a7d-ac37-7cb3c832497a, please check neutron logs for more information.\n'] [ 806.533513] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 806.533790] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance af3ae6a1-44e4-49f7-905c-1480ec3ad0a9.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance af3ae6a1-44e4-49f7-905c-1480ec3ad0a9. [ 806.534493] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] [instance: af3ae6a1-44e4-49f7-905c-1480ec3ad0a9] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance af3ae6a1-44e4-49f7-905c-1480ec3ad0a9. [ 806.577299] nova-conductor[53894]: DEBUG nova.network.neutron [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] [instance: af3ae6a1-44e4-49f7-905c-1480ec3ad0a9] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 806.678735] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Expecting reply to msg 026ac9c10d354f0c8e30fa31da88167b in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 806.836729] nova-conductor[53894]: DEBUG nova.network.neutron [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] [instance: af3ae6a1-44e4-49f7-905c-1480ec3ad0a9] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 806.842350] nova-conductor[53894]: DEBUG nova.network.neutron [None req-4ed7fef4-ba15-454f-8f48-35763a9c12f7 tempest-SecurityGroupsTestJSON-458726018 tempest-SecurityGroupsTestJSON-458726018-project-member] [instance: af3ae6a1-44e4-49f7-905c-1480ec3ad0a9] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 806.844317] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 026ac9c10d354f0c8e30fa31da88167b [ 806.846172] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Took 0.17 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 806.866652] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 806.866885] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 806.867217] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 806.920288] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 806.920530] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 806.920726] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 806.921151] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 806.921295] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 806.921456] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 806.937846] nova-conductor[53894]: DEBUG nova.quota [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Getting quotas for project 3dfb77f12805418eaa6127fc75becec0. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 806.944167] nova-conductor[53894]: DEBUG nova.quota [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Getting quotas for user 789177a2f7be455cadec45cf03d67521 and project 3dfb77f12805418eaa6127fc75becec0. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 806.953260] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 8bf9e7dc-659d-46a3-8016-c09d0c3c0f5a] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 806.953745] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 806.956936] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 806.956936] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 806.957963] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 8bf9e7dc-659d-46a3-8016-c09d0c3c0f5a] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 806.958776] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 806.958958] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 806.959137] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 806.981770] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 806.981770] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 806.981974] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 810.026522] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] [instance: 48b27adc-0bc7-44dd-9330-d0b30593f95f] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 42447106-00fb-4146-a1a8-05ce6aa104cc, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 48b27adc-0bc7-44dd-9330-d0b30593f95f was re-scheduled: Binding failed for port 42447106-00fb-4146-a1a8-05ce6aa104cc, please check neutron logs for more information.\n'] [ 810.029607] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 810.029607] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 48b27adc-0bc7-44dd-9330-d0b30593f95f.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 48b27adc-0bc7-44dd-9330-d0b30593f95f. [ 810.029607] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] [instance: 48b27adc-0bc7-44dd-9330-d0b30593f95f] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 48b27adc-0bc7-44dd-9330-d0b30593f95f. [ 810.053235] nova-conductor[53893]: DEBUG nova.network.neutron [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] [instance: 48b27adc-0bc7-44dd-9330-d0b30593f95f] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 810.118060] nova-conductor[53893]: DEBUG nova.network.neutron [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] [instance: 48b27adc-0bc7-44dd-9330-d0b30593f95f] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 810.122758] nova-conductor[53893]: DEBUG nova.network.neutron [None req-28ebfe0d-c051-4671-adaf-dd6d921582fc tempest-ServerActionsV293TestJSON-1369336445 tempest-ServerActionsV293TestJSON-1369336445-project-member] [instance: 48b27adc-0bc7-44dd-9330-d0b30593f95f] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 810.380978] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Expecting reply to msg 1a5153f27bff4c7d931290b30a0780fd in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 810.575818] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 1a5153f27bff4c7d931290b30a0780fd [ 810.576717] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Took 0.20 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 810.593409] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 810.594299] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 810.594299] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 810.649962] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 810.649962] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 810.649962] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 810.649962] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 810.649962] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 810.650434] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 810.672137] nova-conductor[53894]: DEBUG nova.quota [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Getting quotas for project 18ca4e49b7d7415783135687d8109808. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 810.679555] nova-conductor[53894]: DEBUG nova.quota [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Getting quotas for user 0999e1e5803d4d40bfc1145b45da8085 and project 18ca4e49b7d7415783135687d8109808. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 810.692307] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] [instance: 362c5924-06e1-4385-a1f8-6b0556f7ba8b] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 810.692926] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 810.693156] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 810.693331] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 810.717102] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] [instance: 362c5924-06e1-4385-a1f8-6b0556f7ba8b] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 810.718319] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 810.718319] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 810.718483] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 810.798748] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 810.798748] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 810.800020] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 811.234205] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] [instance: 93ecfc04-23c5-41fc-babf-1a5bd02769b8] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 8b090fba-fb82-4365-b99c-e7d58baf8a68, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 93ecfc04-23c5-41fc-babf-1a5bd02769b8 was re-scheduled: Binding failed for port 8b090fba-fb82-4365-b99c-e7d58baf8a68, please check neutron logs for more information.\n'] [ 811.234784] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 811.235051] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 93ecfc04-23c5-41fc-babf-1a5bd02769b8.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 93ecfc04-23c5-41fc-babf-1a5bd02769b8. [ 811.235359] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] [instance: 93ecfc04-23c5-41fc-babf-1a5bd02769b8] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 93ecfc04-23c5-41fc-babf-1a5bd02769b8. [ 811.259890] nova-conductor[53893]: DEBUG nova.network.neutron [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] [instance: 93ecfc04-23c5-41fc-babf-1a5bd02769b8] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 811.329590] nova-conductor[53893]: DEBUG nova.network.neutron [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] [instance: 93ecfc04-23c5-41fc-babf-1a5bd02769b8] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 811.331713] nova-conductor[53893]: DEBUG nova.network.neutron [None req-2b31308d-4f3c-47db-90b3-3b2f0b221688 tempest-ServersV294TestFqdnHostnames-1183918085 tempest-ServersV294TestFqdnHostnames-1183918085-project-member] [instance: 93ecfc04-23c5-41fc-babf-1a5bd02769b8] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 812.266354] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] [instance: 07d2f1e7-c08e-434c-aea7-941ef75f16ba] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn\n self._fetch_image_if_missing(context, vi)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing\n image_cache(vi, tmp_image_ds_loc)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image\n vm_util.copy_virtual_disk(\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk\n session._wait_for_task(vmdk_copy_task)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task\n return self.wait_for_task(task_ref)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task\n return evt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner\n self.f(*self.args, **self.kw)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task\n raise exceptions.translate_fault(task_info.error)\n', "oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n", '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', "nova.exception.RescheduledException: Build of instance 07d2f1e7-c08e-434c-aea7-941ef75f16ba was re-scheduled: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n"] [ 812.266926] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 812.267167] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 07d2f1e7-c08e-434c-aea7-941ef75f16ba.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 07d2f1e7-c08e-434c-aea7-941ef75f16ba. [ 812.267485] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-85685b95-ee2c-4253-83c5-dec83d6d5968 tempest-ServersListShow296Test-863923347 tempest-ServersListShow296Test-863923347-project-member] [instance: 07d2f1e7-c08e-434c-aea7-941ef75f16ba] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 07d2f1e7-c08e-434c-aea7-941ef75f16ba. [ 812.709097] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell0:SELECT=8 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 812.748167] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=36,nova_cell1:INSERT=78,nova_cell1:SAVEPOINT=2,nova_cell1:RELEASE=2 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 812.753414] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=44,nova_api:UPDATE=3,nova_api:DELETE=3 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 812.758839] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=52,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 813.040282] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: e600280c-2414-420d-bc8d-6e3e7979fccf] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 99a82fe7-f10a-4919-91d9-fdde7ed0a044, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance e600280c-2414-420d-bc8d-6e3e7979fccf was re-scheduled: Binding failed for port 99a82fe7-f10a-4919-91d9-fdde7ed0a044, please check neutron logs for more information.\n'] [ 813.041609] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 813.041609] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e600280c-2414-420d-bc8d-6e3e7979fccf.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e600280c-2414-420d-bc8d-6e3e7979fccf. [ 813.041609] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: e600280c-2414-420d-bc8d-6e3e7979fccf] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance e600280c-2414-420d-bc8d-6e3e7979fccf. [ 813.080017] nova-conductor[53893]: DEBUG nova.network.neutron [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: e600280c-2414-420d-bc8d-6e3e7979fccf] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 813.152389] nova-conductor[53893]: DEBUG nova.network.neutron [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: e600280c-2414-420d-bc8d-6e3e7979fccf] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 813.155740] nova-conductor[53893]: DEBUG nova.network.neutron [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: e600280c-2414-420d-bc8d-6e3e7979fccf] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 813.754422] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 7cf10fb8-e3db-44fb-b9c3-4b9bb92d18e3] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port d3a45c31-7ab7-489a-aac0-8e6c223de0e7, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 7cf10fb8-e3db-44fb-b9c3-4b9bb92d18e3 was re-scheduled: Binding failed for port d3a45c31-7ab7-489a-aac0-8e6c223de0e7, please check neutron logs for more information.\n'] [ 813.755857] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 813.755857] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7cf10fb8-e3db-44fb-b9c3-4b9bb92d18e3.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7cf10fb8-e3db-44fb-b9c3-4b9bb92d18e3. [ 813.756442] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 7cf10fb8-e3db-44fb-b9c3-4b9bb92d18e3] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7cf10fb8-e3db-44fb-b9c3-4b9bb92d18e3. [ 813.792211] nova-conductor[53893]: DEBUG nova.network.neutron [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 7cf10fb8-e3db-44fb-b9c3-4b9bb92d18e3] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 813.863704] nova-conductor[53893]: DEBUG nova.network.neutron [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 7cf10fb8-e3db-44fb-b9c3-4b9bb92d18e3] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 813.871804] nova-conductor[53893]: DEBUG nova.network.neutron [None req-b4aa60c9-059c-4b71-b748-886528a0ce8f tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 7cf10fb8-e3db-44fb-b9c3-4b9bb92d18e3] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 814.378381] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] [instance: 500259b0-a57d-43ff-9c88-46190b6a3a10] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port e4b7d217-6d21-4ae5-a1f4-cbf7611ff2be, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 500259b0-a57d-43ff-9c88-46190b6a3a10 was re-scheduled: Binding failed for port e4b7d217-6d21-4ae5-a1f4-cbf7611ff2be, please check neutron logs for more information.\n'] [ 814.378803] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 814.378803] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 500259b0-a57d-43ff-9c88-46190b6a3a10.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 500259b0-a57d-43ff-9c88-46190b6a3a10. [ 814.379253] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] [instance: 500259b0-a57d-43ff-9c88-46190b6a3a10] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 500259b0-a57d-43ff-9c88-46190b6a3a10. [ 814.418818] nova-conductor[53894]: DEBUG nova.network.neutron [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] [instance: 500259b0-a57d-43ff-9c88-46190b6a3a10] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 814.475353] nova-conductor[53894]: DEBUG nova.network.neutron [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] [instance: 500259b0-a57d-43ff-9c88-46190b6a3a10] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 814.483075] nova-conductor[53894]: DEBUG nova.network.neutron [None req-60532300-c080-4475-a503-9720cbbac502 tempest-ServerPasswordTestJSON-1155626657 tempest-ServerPasswordTestJSON-1155626657-project-member] [instance: 500259b0-a57d-43ff-9c88-46190b6a3a10] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 814.778492] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: 4ad1a310-5786-4bb6-87ff-72069f692eff] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port c030586c-57a9-4ed0-bfb3-955b0961e165, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 4ad1a310-5786-4bb6-87ff-72069f692eff was re-scheduled: Binding failed for port c030586c-57a9-4ed0-bfb3-955b0961e165, please check neutron logs for more information.\n'] [ 814.778492] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 814.778492] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 4ad1a310-5786-4bb6-87ff-72069f692eff.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 4ad1a310-5786-4bb6-87ff-72069f692eff. [ 814.778700] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: 4ad1a310-5786-4bb6-87ff-72069f692eff] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 4ad1a310-5786-4bb6-87ff-72069f692eff. [ 814.809136] nova-conductor[53894]: DEBUG nova.network.neutron [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: 4ad1a310-5786-4bb6-87ff-72069f692eff] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 814.885287] nova-conductor[53894]: DEBUG nova.network.neutron [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: 4ad1a310-5786-4bb6-87ff-72069f692eff] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 814.890664] nova-conductor[53894]: DEBUG nova.network.neutron [None req-a2687786-2462-4cea-aa9e-693ff5d747d9 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: 4ad1a310-5786-4bb6-87ff-72069f692eff] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 815.614946] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] [instance: f7d0db3d-9247-409a-bf1e-4b53c2368ddc] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port f32cbb9a-7422-4f1e-b576-b28aa8cc4371, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance f7d0db3d-9247-409a-bf1e-4b53c2368ddc was re-scheduled: Binding failed for port f32cbb9a-7422-4f1e-b576-b28aa8cc4371, please check neutron logs for more information.\n'] [ 815.616680] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 815.617074] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance f7d0db3d-9247-409a-bf1e-4b53c2368ddc.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance f7d0db3d-9247-409a-bf1e-4b53c2368ddc. [ 815.617433] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] [instance: f7d0db3d-9247-409a-bf1e-4b53c2368ddc] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance f7d0db3d-9247-409a-bf1e-4b53c2368ddc. [ 815.647680] nova-conductor[53894]: DEBUG nova.network.neutron [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] [instance: f7d0db3d-9247-409a-bf1e-4b53c2368ddc] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 815.690726] nova-conductor[53894]: DEBUG nova.network.neutron [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] [instance: f7d0db3d-9247-409a-bf1e-4b53c2368ddc] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 815.695374] nova-conductor[53894]: DEBUG nova.network.neutron [None req-30746edb-2bbc-445d-bd61-e88a0aad8845 tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] [instance: f7d0db3d-9247-409a-bf1e-4b53c2368ddc] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 816.557726] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Expecting reply to msg 5db0bc9851014055a059ac0419aea75c in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 816.732447] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5db0bc9851014055a059ac0419aea75c [ 816.733312] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Took 0.18 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 816.750582] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 816.751017] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 816.752027] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 816.800327] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 816.800327] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 816.800479] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 816.800824] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 816.801008] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 816.801200] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 816.814564] nova-conductor[53894]: DEBUG nova.quota [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Getting quotas for project 9dbc2efe50ed4ae5a5f0cf6f492a20bf. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 816.819161] nova-conductor[53894]: DEBUG nova.quota [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Getting quotas for user 7ce2159a0ccb46c89a7574d04142e926 and project 9dbc2efe50ed4ae5a5f0cf6f492a20bf. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 816.826356] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: 9c6c9639-614f-4ffa-a6db-e70ef37b9954] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 816.827129] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 816.827345] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 816.827512] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 816.834287] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: 9c6c9639-614f-4ffa-a6db-e70ef37b9954] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 816.834287] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 816.834287] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 816.834436] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 816.864975] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 816.865265] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 816.865445] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 817.176407] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Expecting reply to msg fc0bcba12e8b48ffb10b39c1d1f57d6d in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 817.340710] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg fc0bcba12e8b48ffb10b39c1d1f57d6d [ 817.342354] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Took 0.17 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 817.367682] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 817.367909] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 817.368093] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 817.448979] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 817.448979] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 817.448979] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 817.449268] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 817.453020] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 817.453020] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 817.474080] nova-conductor[53894]: DEBUG nova.quota [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Getting quotas for project bff3be1976444e58a2b7be93d47f50ef. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 817.478029] nova-conductor[53894]: DEBUG nova.quota [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Getting quotas for user 84f05a61e0f6499bb35c44d254226249 and project bff3be1976444e58a2b7be93d47f50ef. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 817.488345] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: bbd46264-8992-4e56-9896-675500fe587b] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 817.489114] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 817.489114] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 817.489252] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 817.500163] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: bbd46264-8992-4e56-9896-675500fe587b] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 817.500955] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 817.501271] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 817.501423] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 817.530271] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 817.530735] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 817.533017] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 818.843024] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] [instance: 362c5924-06e1-4385-a1f8-6b0556f7ba8b] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port b183ba56-040d-4b47-8d9f-2483248eb675, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 362c5924-06e1-4385-a1f8-6b0556f7ba8b was re-scheduled: Binding failed for port b183ba56-040d-4b47-8d9f-2483248eb675, please check neutron logs for more information.\n'] [ 818.843448] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 818.843757] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 362c5924-06e1-4385-a1f8-6b0556f7ba8b.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 362c5924-06e1-4385-a1f8-6b0556f7ba8b. [ 818.843849] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] [instance: 362c5924-06e1-4385-a1f8-6b0556f7ba8b] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 362c5924-06e1-4385-a1f8-6b0556f7ba8b. [ 818.874586] nova-conductor[53893]: DEBUG nova.network.neutron [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] [instance: 362c5924-06e1-4385-a1f8-6b0556f7ba8b] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 818.951157] nova-conductor[53893]: DEBUG nova.network.neutron [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] [instance: 362c5924-06e1-4385-a1f8-6b0556f7ba8b] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 818.958860] nova-conductor[53893]: DEBUG nova.network.neutron [None req-604e9287-9bb1-43fc-976a-96f46dd10c4c tempest-ServersTestFqdnHostnames-1549182594 tempest-ServersTestFqdnHostnames-1549182594-project-member] [instance: 362c5924-06e1-4385-a1f8-6b0556f7ba8b] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 819.948932] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Expecting reply to msg aa90712284c6437091c0794062614ded in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 820.038199] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Expecting reply to msg d70332b31a2543c986b9f636754147ff in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 820.146786] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aa90712284c6437091c0794062614ded [ 820.150548] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Took 0.20 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 820.169526] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.169526] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.169526] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 820.217471] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d70332b31a2543c986b9f636754147ff [ 820.218093] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Took 0.18 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 820.230061] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.230061] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.230502] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 820.230867] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.230867] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.231329] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 820.250355] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.250641] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.251211] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 820.254346] nova-conductor[53894]: DEBUG nova.quota [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Getting quotas for project 57787a709d744ea4a19a2cfb923d89d4. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 820.261351] nova-conductor[53894]: DEBUG nova.quota [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Getting quotas for user 2d6b88b1f4394a63b32be00fc9dcc761 and project 57787a709d744ea4a19a2cfb923d89d4. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 820.273155] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: 848010fd-76ed-43d9-8d74-62b09062a2b6] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 820.273663] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.273864] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.274700] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 820.288833] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: 848010fd-76ed-43d9-8d74-62b09062a2b6] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 820.289623] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.289925] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.292952] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 820.317576] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.317576] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.317744] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 820.318185] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.318401] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.318536] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 820.328024] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.328218] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.328388] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 820.337970] nova-conductor[53894]: DEBUG nova.quota [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Getting quotas for project ae1347959e924d97b351fdd7c40c4777. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 820.340843] nova-conductor[53894]: DEBUG nova.quota [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Getting quotas for user e5a10b199013411d80830a41a0abb711 and project ae1347959e924d97b351fdd7c40c4777. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 820.358614] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] [instance: 7cacf5b7-500d-493c-a86a-2a8a03cc6eac] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 820.358995] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.359930] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.359930] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 820.368018] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] [instance: 7cacf5b7-500d-493c-a86a-2a8a03cc6eac] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 820.368018] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.368018] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.368018] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 820.425817] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 820.425964] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 820.426456] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 821.618638] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 8bf9e7dc-659d-46a3-8016-c09d0c3c0f5a] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 24dbe2dd-6847-4ffd-9ba1-9ea5207dd415, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 8bf9e7dc-659d-46a3-8016-c09d0c3c0f5a was re-scheduled: Binding failed for port 24dbe2dd-6847-4ffd-9ba1-9ea5207dd415, please check neutron logs for more information.\n'] [ 821.619260] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 821.619499] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8bf9e7dc-659d-46a3-8016-c09d0c3c0f5a.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8bf9e7dc-659d-46a3-8016-c09d0c3c0f5a. [ 821.619811] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 8bf9e7dc-659d-46a3-8016-c09d0c3c0f5a] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 8bf9e7dc-659d-46a3-8016-c09d0c3c0f5a. [ 821.655210] nova-conductor[53894]: DEBUG nova.network.neutron [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 8bf9e7dc-659d-46a3-8016-c09d0c3c0f5a] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 821.745616] nova-conductor[53894]: DEBUG nova.network.neutron [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 8bf9e7dc-659d-46a3-8016-c09d0c3c0f5a] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 821.749699] nova-conductor[53894]: DEBUG nova.network.neutron [None req-f366b3e9-58eb-46da-a72b-978eed57d136 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 8bf9e7dc-659d-46a3-8016-c09d0c3c0f5a] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 824.152480] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Expecting reply to msg 88b03af4bf1b43818f5bdee9c94b9def in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 824.304107] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 88b03af4bf1b43818f5bdee9c94b9def [ 824.304921] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Took 0.16 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 824.322415] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 824.322765] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 824.323019] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 824.371998] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 824.372328] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 824.372559] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 824.373252] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 824.373527] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 824.373741] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 824.388941] nova-conductor[53894]: DEBUG nova.quota [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Getting quotas for project b54f5acf42c64133afbb208929492c31. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 824.392393] nova-conductor[53894]: DEBUG nova.quota [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Getting quotas for user 06291742abbe4dee8092fba657b8ab91 and project b54f5acf42c64133afbb208929492c31. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 824.399443] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 382b96fa-d807-4f96-b47e-784dfcd26437] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 824.399994] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 824.400508] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 824.400751] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 824.407433] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 382b96fa-d807-4f96-b47e-784dfcd26437] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 824.409073] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 824.409073] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 824.409073] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 824.438034] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 824.438325] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 824.438588] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 826.078915] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Expecting reply to msg 240cb8acbcc245048527a092f4acaf9a in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 826.309263] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 240cb8acbcc245048527a092f4acaf9a [ 826.310186] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Took 0.23 seconds to select destinations for 2 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 826.327730] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 826.327958] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 826.328144] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 826.375318] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 826.375498] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 826.375668] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 826.404603] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 826.404848] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 826.405029] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 826.405409] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 826.405594] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 826.405757] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 826.417213] nova-conductor[53894]: DEBUG nova.quota [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Getting quotas for project 201633229817424689c8fe5557911ae9. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 826.420773] nova-conductor[53894]: DEBUG nova.quota [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Getting quotas for user 185a047edb6241d59feb8ad5d2bbc749 and project 201633229817424689c8fe5557911ae9. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 826.426697] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: bea7629d-c264-457f-b887-443f1ada1e9c] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 826.427460] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 826.427460] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 826.427666] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 826.434559] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: bea7629d-c264-457f-b887-443f1ada1e9c] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 826.437929] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 826.437929] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 826.437929] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 826.465189] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 826.465419] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 826.465590] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 826.472728] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: 49d32ccb-b1fd-4640-bc6f-8c08f9df0423] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 826.473221] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 826.473426] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 826.473623] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 826.490913] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: 49d32ccb-b1fd-4640-bc6f-8c08f9df0423] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 826.491765] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 826.491885] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 826.492087] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 826.533539] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 826.533539] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 826.533539] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 828.892854] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Expecting reply to msg 5486480dbd594f268c9c0f203dd58715 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 828.935065] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: 9c6c9639-614f-4ffa-a6db-e70ef37b9954] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 55011e89-cff6-4abb-9a03-2344a1163928, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 9c6c9639-614f-4ffa-a6db-e70ef37b9954 was re-scheduled: Binding failed for port 55011e89-cff6-4abb-9a03-2344a1163928, please check neutron logs for more information.\n'] [ 828.935391] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 828.935652] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 9c6c9639-614f-4ffa-a6db-e70ef37b9954.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 9c6c9639-614f-4ffa-a6db-e70ef37b9954. [ 828.936258] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: 9c6c9639-614f-4ffa-a6db-e70ef37b9954] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 9c6c9639-614f-4ffa-a6db-e70ef37b9954. [ 828.965022] nova-conductor[53894]: DEBUG nova.network.neutron [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: 9c6c9639-614f-4ffa-a6db-e70ef37b9954] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 829.047042] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5486480dbd594f268c9c0f203dd58715 [ 829.047884] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Took 0.16 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 829.067362] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 829.067652] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 829.067770] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 829.106670] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 829.106874] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 829.107065] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 829.107725] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 829.108039] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 829.108093] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 829.119456] nova-conductor[53893]: DEBUG nova.quota [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Getting quotas for project 3dfb77f12805418eaa6127fc75becec0. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 829.122180] nova-conductor[53893]: DEBUG nova.quota [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Getting quotas for user 789177a2f7be455cadec45cf03d67521 and project 3dfb77f12805418eaa6127fc75becec0. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 829.131094] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: cb22b0a4-bcef-4964-bd63-5abb2789cedd] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 829.131596] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 829.131734] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 829.131897] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 829.135280] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: cb22b0a4-bcef-4964-bd63-5abb2789cedd] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 829.135918] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 829.136260] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 829.136435] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 829.155587] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 829.155951] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 829.155951] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 829.260236] nova-conductor[53894]: DEBUG nova.network.neutron [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: 9c6c9639-614f-4ffa-a6db-e70ef37b9954] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 829.264245] nova-conductor[53894]: DEBUG nova.network.neutron [None req-bc4e3c3b-6f00-4137-aa0a-06075495cf98 tempest-ImagesTestJSON-873357144 tempest-ImagesTestJSON-873357144-project-member] [instance: 9c6c9639-614f-4ffa-a6db-e70ef37b9954] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 829.464958] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: bbd46264-8992-4e56-9896-675500fe587b] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port fbd834cc-c82f-429f-889a-1b197f4186b6, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance bbd46264-8992-4e56-9896-675500fe587b was re-scheduled: Binding failed for port fbd834cc-c82f-429f-889a-1b197f4186b6, please check neutron logs for more information.\n'] [ 829.467995] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 829.468443] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance bbd46264-8992-4e56-9896-675500fe587b.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance bbd46264-8992-4e56-9896-675500fe587b. [ 829.468902] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: bbd46264-8992-4e56-9896-675500fe587b] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance bbd46264-8992-4e56-9896-675500fe587b. [ 829.507027] nova-conductor[53893]: DEBUG nova.network.neutron [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: bbd46264-8992-4e56-9896-675500fe587b] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 829.596247] nova-conductor[53893]: DEBUG nova.network.neutron [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: bbd46264-8992-4e56-9896-675500fe587b] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 829.600299] nova-conductor[53893]: DEBUG nova.network.neutron [None req-e4e8ec04-7676-499a-bfd8-d29b512d5166 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: bbd46264-8992-4e56-9896-675500fe587b] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 830.167485] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Expecting reply to msg aab173e4c4e4406b8479b1d947ffc4ed in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 830.330649] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg aab173e4c4e4406b8479b1d947ffc4ed [ 830.331692] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Took 0.17 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 830.346431] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 830.347062] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 830.347062] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 830.403471] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 830.403795] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 830.404860] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 830.404860] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 830.405072] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 830.405320] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 830.416139] nova-conductor[53893]: DEBUG nova.quota [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Getting quotas for project 63bee6818d034676b755bb90af6015a2. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 830.421739] nova-conductor[53893]: DEBUG nova.quota [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Getting quotas for user fcdf7d6ebc814fc0b579eb2b98ef5a92 and project 63bee6818d034676b755bb90af6015a2. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 830.429381] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] [instance: ad2881a4-9715-40ed-8489-85a6a575fb30] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 830.430256] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 830.430496] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 830.432016] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 830.447578] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] [instance: ad2881a4-9715-40ed-8489-85a6a575fb30] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 830.448345] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 830.448572] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 830.448735] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 830.495412] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 830.499020] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 830.499020] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 831.194817] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Expecting reply to msg 5bf46a1a97cd46ff97bc74c82b8cd3ca in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 831.376361] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 5bf46a1a97cd46ff97bc74c82b8cd3ca [ 831.377227] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Took 0.19 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 831.401217] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 831.401217] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 831.401942] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 831.472231] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 831.472555] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 831.472879] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 831.473388] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 831.473634] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 831.473918] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 831.495352] nova-conductor[53894]: DEBUG nova.quota [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Getting quotas for project a371901b35c94dd6a97437f6d2f0ab57. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 831.500913] nova-conductor[53894]: DEBUG nova.quota [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Getting quotas for user a9ed2b40336d4343b4fea090b06ab970 and project a371901b35c94dd6a97437f6d2f0ab57. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 831.510018] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] [instance: d364fc2e-89d7-4b2e-a510-19148a8f1a2b] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 831.510018] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 831.510018] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 831.510018] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 831.517903] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] [instance: d364fc2e-89d7-4b2e-a510-19148a8f1a2b] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 831.518620] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 831.518827] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 831.518992] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 831.547971] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 831.548243] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 831.548421] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 833.113445] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: 848010fd-76ed-43d9-8d74-62b09062a2b6] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 64a4ae94-4388-4b11-ab1e-f234740d4680, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 848010fd-76ed-43d9-8d74-62b09062a2b6 was re-scheduled: Binding failed for port 64a4ae94-4388-4b11-ab1e-f234740d4680, please check neutron logs for more information.\n'] [ 833.114080] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 833.115449] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 848010fd-76ed-43d9-8d74-62b09062a2b6.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 848010fd-76ed-43d9-8d74-62b09062a2b6. [ 833.115449] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: 848010fd-76ed-43d9-8d74-62b09062a2b6] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 848010fd-76ed-43d9-8d74-62b09062a2b6. [ 833.143783] nova-conductor[53893]: DEBUG nova.network.neutron [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: 848010fd-76ed-43d9-8d74-62b09062a2b6] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 833.221165] nova-conductor[53893]: DEBUG nova.network.neutron [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: 848010fd-76ed-43d9-8d74-62b09062a2b6] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 833.225039] nova-conductor[53893]: DEBUG nova.network.neutron [None req-21091e0f-affd-42c7-b11a-5c1c6972e824 tempest-AttachInterfacesTestJSON-2048770672 tempest-AttachInterfacesTestJSON-2048770672-project-member] [instance: 848010fd-76ed-43d9-8d74-62b09062a2b6] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 833.472369] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Expecting reply to msg f741d0519bcc4f099a793db2c2f23df1 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 833.656359] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg f741d0519bcc4f099a793db2c2f23df1 [ 833.657171] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Took 0.19 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 833.686234] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 833.686478] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 833.686792] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 833.742082] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 833.742307] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 833.742495] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 833.742938] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 833.743185] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 833.743387] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 833.755049] nova-conductor[53893]: DEBUG nova.quota [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Getting quotas for project a371901b35c94dd6a97437f6d2f0ab57. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 833.760115] nova-conductor[53893]: DEBUG nova.quota [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Getting quotas for user a9ed2b40336d4343b4fea090b06ab970 and project a371901b35c94dd6a97437f6d2f0ab57. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 833.766836] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] [instance: dc31ebf5-889b-438b-9f54-6df807714a38] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 833.766836] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 833.766836] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 833.766836] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 833.771586] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] [instance: dc31ebf5-889b-438b-9f54-6df807714a38] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 833.772280] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 833.772474] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 833.772633] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 833.793696] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 833.793919] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 833.794102] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 835.389597] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Expecting reply to msg 59f037efb20a4ab191b8aa9a9882dc7d in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 835.559189] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 59f037efb20a4ab191b8aa9a9882dc7d [ 835.559189] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Took 0.18 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 835.578031] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 835.578031] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 835.578031] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 835.621052] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 835.621337] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 835.621514] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 835.621933] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 835.622129] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 835.622290] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 835.638078] nova-conductor[53894]: DEBUG nova.quota [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Getting quotas for project b7300699b4cb49de89d096e3a1ac1778. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 835.650808] nova-conductor[53894]: DEBUG nova.quota [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Getting quotas for user bcf95f76d0f54cbebde1301f62b44bcd and project b7300699b4cb49de89d096e3a1ac1778. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 835.656771] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] [instance: 1c342a36-b05d-452e-bbe1-fedf93e9f9d1] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 835.657298] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 835.657504] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 835.657672] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 835.664113] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] [instance: 1c342a36-b05d-452e-bbe1-fedf93e9f9d1] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 835.664820] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 835.665034] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 835.665207] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 835.685297] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 835.685642] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 835.685762] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 836.276771] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] [instance: 7cacf5b7-500d-493c-a86a-2a8a03cc6eac] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 87f8c98f-4a27-42fc-b13f-d8e6af9b323a, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 7cacf5b7-500d-493c-a86a-2a8a03cc6eac was re-scheduled: Binding failed for port 87f8c98f-4a27-42fc-b13f-d8e6af9b323a, please check neutron logs for more information.\n'] [ 836.277453] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 836.277697] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7cacf5b7-500d-493c-a86a-2a8a03cc6eac.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7cacf5b7-500d-493c-a86a-2a8a03cc6eac. [ 836.278012] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] [instance: 7cacf5b7-500d-493c-a86a-2a8a03cc6eac] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 7cacf5b7-500d-493c-a86a-2a8a03cc6eac. [ 836.326702] nova-conductor[53893]: DEBUG nova.network.neutron [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] [instance: 7cacf5b7-500d-493c-a86a-2a8a03cc6eac] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 836.521570] nova-conductor[53893]: DEBUG nova.network.neutron [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] [instance: 7cacf5b7-500d-493c-a86a-2a8a03cc6eac] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 836.535009] nova-conductor[53893]: DEBUG nova.network.neutron [None req-4408f30b-b452-4784-99a2-f2e10239d43e tempest-InstanceActionsNegativeTestJSON-168004218 tempest-InstanceActionsNegativeTestJSON-168004218-project-member] [instance: 7cacf5b7-500d-493c-a86a-2a8a03cc6eac] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 837.322553] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Expecting reply to msg 6b4fa9b4b65c499d9d430c11d1d0f03f in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 837.534040] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 6b4fa9b4b65c499d9d430c11d1d0f03f [ 837.534750] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Took 0.22 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 837.608015] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 837.608264] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 837.608472] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 837.659609] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 837.660280] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 837.660563] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 837.661040] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 837.661288] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 837.661528] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 837.679135] nova-conductor[53893]: DEBUG nova.quota [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Getting quotas for project bff3be1976444e58a2b7be93d47f50ef. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 837.685929] nova-conductor[53893]: DEBUG nova.quota [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Getting quotas for user 84f05a61e0f6499bb35c44d254226249 and project bff3be1976444e58a2b7be93d47f50ef. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 837.693771] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: b21f2e22-a6d6-4ab5-baa7-110004aa776e] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 837.694030] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 837.694030] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 837.694030] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 837.697865] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: b21f2e22-a6d6-4ab5-baa7-110004aa776e] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 837.698637] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 837.698976] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 837.699187] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 837.716490] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 382b96fa-d807-4f96-b47e-784dfcd26437] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 51ba6a99-29fe-431c-bc59-906ac6340e1a, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 382b96fa-d807-4f96-b47e-784dfcd26437 was re-scheduled: Binding failed for port 51ba6a99-29fe-431c-bc59-906ac6340e1a, please check neutron logs for more information.\n'] [ 837.716490] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 837.716490] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 382b96fa-d807-4f96-b47e-784dfcd26437.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 382b96fa-d807-4f96-b47e-784dfcd26437. [ 837.716490] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 382b96fa-d807-4f96-b47e-784dfcd26437] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 382b96fa-d807-4f96-b47e-784dfcd26437. [ 837.740340] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 837.743038] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 837.743038] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 837.755281] nova-conductor[53893]: DEBUG nova.network.neutron [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 382b96fa-d807-4f96-b47e-784dfcd26437] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 837.827423] nova-conductor[53893]: DEBUG nova.network.neutron [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 382b96fa-d807-4f96-b47e-784dfcd26437] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 837.832838] nova-conductor[53893]: DEBUG nova.network.neutron [None req-c0015503-e2c1-4e56-9631-c2c98cca1bc3 tempest-AttachVolumeNegativeTest-427402148 tempest-AttachVolumeNegativeTest-427402148-project-member] [instance: 382b96fa-d807-4f96-b47e-784dfcd26437] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 839.643790] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: 49d32ccb-b1fd-4640-bc6f-8c08f9df0423] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 24404798-6dd9-490d-817a-2a8df00a98ed, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 49d32ccb-b1fd-4640-bc6f-8c08f9df0423 was re-scheduled: Binding failed for port 24404798-6dd9-490d-817a-2a8df00a98ed, please check neutron logs for more information.\n'] [ 839.644458] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 839.644696] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 49d32ccb-b1fd-4640-bc6f-8c08f9df0423.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 49d32ccb-b1fd-4640-bc6f-8c08f9df0423. [ 839.645025] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: 49d32ccb-b1fd-4640-bc6f-8c08f9df0423] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 49d32ccb-b1fd-4640-bc6f-8c08f9df0423. [ 839.674742] nova-conductor[53894]: DEBUG nova.network.neutron [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: 49d32ccb-b1fd-4640-bc6f-8c08f9df0423] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 839.690369] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: bea7629d-c264-457f-b887-443f1ada1e9c] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 18c8641b-7198-4e85-a2b7-ddf8a76c3639, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance bea7629d-c264-457f-b887-443f1ada1e9c was re-scheduled: Binding failed for port 18c8641b-7198-4e85-a2b7-ddf8a76c3639, please check neutron logs for more information.\n'] [ 839.691047] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 839.691323] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance bea7629d-c264-457f-b887-443f1ada1e9c.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance bea7629d-c264-457f-b887-443f1ada1e9c. [ 839.691729] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: bea7629d-c264-457f-b887-443f1ada1e9c] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance bea7629d-c264-457f-b887-443f1ada1e9c. [ 839.716240] nova-conductor[53893]: DEBUG nova.network.neutron [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: bea7629d-c264-457f-b887-443f1ada1e9c] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 839.729483] nova-conductor[53894]: DEBUG nova.network.neutron [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: 49d32ccb-b1fd-4640-bc6f-8c08f9df0423] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 839.737036] nova-conductor[53894]: DEBUG nova.network.neutron [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: 49d32ccb-b1fd-4640-bc6f-8c08f9df0423] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 839.767920] nova-conductor[53893]: DEBUG nova.network.neutron [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: bea7629d-c264-457f-b887-443f1ada1e9c] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 839.774250] nova-conductor[53893]: DEBUG nova.network.neutron [None req-82fa6a67-38bd-452b-b5c0-5a0b3db0fda8 tempest-MultipleCreateTestJSON-962420962 tempest-MultipleCreateTestJSON-962420962-project-member] [instance: bea7629d-c264-457f-b887-443f1ada1e9c] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 839.883622] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 839.884350] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 839.884350] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 839.917189] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Expecting reply to msg c0b65487ecc84c0694719e1548284790 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 840.099222] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c0b65487ecc84c0694719e1548284790 [ 840.100039] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Took 0.19 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 840.122528] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 840.123265] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 840.123594] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.001s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 840.183385] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 840.183703] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 840.183777] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 840.185042] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 840.185042] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 840.185042] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 840.197535] nova-conductor[53894]: DEBUG nova.quota [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Getting quotas for project 628c7ec13bef4d04b411f7ef6e39ef0f. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 840.201810] nova-conductor[53894]: DEBUG nova.quota [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Getting quotas for user f29927aa10404757aeb7596fc8ea9597 and project 628c7ec13bef4d04b411f7ef6e39ef0f. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 840.206926] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] [instance: 5991598e-20cd-475d-bbdc-83cd4909a31e] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 840.207452] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 840.207651] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 840.207810] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 840.215363] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] [instance: 5991598e-20cd-475d-bbdc-83cd4909a31e] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 840.215363] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 840.215363] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 840.215641] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 840.237306] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 840.237521] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 840.237688] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 841.035736] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: cb22b0a4-bcef-4964-bd63-5abb2789cedd] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 7ed6abdf-02e3-4eb2-89c4-fe379e8042ad, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance cb22b0a4-bcef-4964-bd63-5abb2789cedd was re-scheduled: Binding failed for port 7ed6abdf-02e3-4eb2-89c4-fe379e8042ad, please check neutron logs for more information.\n'] [ 841.036827] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 841.037381] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance cb22b0a4-bcef-4964-bd63-5abb2789cedd.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance cb22b0a4-bcef-4964-bd63-5abb2789cedd. [ 841.038424] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: cb22b0a4-bcef-4964-bd63-5abb2789cedd] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance cb22b0a4-bcef-4964-bd63-5abb2789cedd. [ 841.064570] nova-conductor[53893]: DEBUG nova.network.neutron [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: cb22b0a4-bcef-4964-bd63-5abb2789cedd] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 841.166274] nova-conductor[53893]: DEBUG nova.network.neutron [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: cb22b0a4-bcef-4964-bd63-5abb2789cedd] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 841.168938] nova-conductor[53893]: DEBUG nova.network.neutron [None req-7a7d437f-2932-41bc-9bad-6754e8578037 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: cb22b0a4-bcef-4964-bd63-5abb2789cedd] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 844.697974] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Expecting reply to msg 27293e7ff27e45adaf9a6bcfbfe83adb in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 844.821691] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] [instance: ad2881a4-9715-40ed-8489-85a6a575fb30] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 6fd9c64d-1ba4-44d6-9b86-1a09fba34bc4, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance ad2881a4-9715-40ed-8489-85a6a575fb30 was re-scheduled: Binding failed for port 6fd9c64d-1ba4-44d6-9b86-1a09fba34bc4, please check neutron logs for more information.\n'] [ 844.822528] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 844.823236] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ad2881a4-9715-40ed-8489-85a6a575fb30.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ad2881a4-9715-40ed-8489-85a6a575fb30. [ 844.823802] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] [instance: ad2881a4-9715-40ed-8489-85a6a575fb30] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ad2881a4-9715-40ed-8489-85a6a575fb30. [ 844.852058] nova-conductor[53893]: DEBUG nova.network.neutron [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] [instance: ad2881a4-9715-40ed-8489-85a6a575fb30] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 844.859807] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 27293e7ff27e45adaf9a6bcfbfe83adb [ 844.860625] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Took 0.17 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 844.883618] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 844.883867] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 844.884048] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 844.926193] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 844.926502] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 844.926692] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 844.927088] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 844.927268] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 844.927434] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 844.947941] nova-conductor[53893]: DEBUG nova.quota [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Getting quotas for project 3dfb77f12805418eaa6127fc75becec0. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 844.952315] nova-conductor[53893]: DEBUG nova.quota [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Getting quotas for user 789177a2f7be455cadec45cf03d67521 and project 3dfb77f12805418eaa6127fc75becec0. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 844.958871] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 88a6495c-b9b7-4822-9cae-f27594545b3e] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 844.959648] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 844.959847] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 844.960020] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 844.964589] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 88a6495c-b9b7-4822-9cae-f27594545b3e] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 844.965347] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 844.965547] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 844.965739] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 844.996353] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 844.996622] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 844.996798] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 845.125303] nova-conductor[53893]: DEBUG nova.network.neutron [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] [instance: ad2881a4-9715-40ed-8489-85a6a575fb30] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 845.130048] nova-conductor[53893]: DEBUG nova.network.neutron [None req-7ee73e40-d081-4f4d-ad4f-c1e96e36bd29 tempest-ServersNegativeTestMultiTenantJSON-1155338673 tempest-ServersNegativeTestMultiTenantJSON-1155338673-project-member] [instance: ad2881a4-9715-40ed-8489-85a6a575fb30] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 845.992405] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Expecting reply to msg 89c9e8f71d2c4eb194c5922457c5c6bb in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 846.145716] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 89c9e8f71d2c4eb194c5922457c5c6bb [ 846.146635] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Took 0.16 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 846.168254] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 846.168525] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 846.168707] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 846.212505] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 846.212723] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 846.212892] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 846.213261] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 846.213438] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 846.214010] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 846.222758] nova-conductor[53894]: DEBUG nova.quota [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Getting quotas for project 88f0dbb8d2bc4a0d9e1c976860700368. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 846.226112] nova-conductor[53894]: DEBUG nova.quota [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Getting quotas for user 892a851386304c3f8145b5494775148e and project 88f0dbb8d2bc4a0d9e1c976860700368. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 846.232171] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] [instance: 0f6368a9-cadc-46b4-be16-017724580876] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 846.232642] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 846.233214] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 846.233214] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 846.237106] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] [instance: 0f6368a9-cadc-46b4-be16-017724580876] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 846.237746] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 846.240830] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 846.240830] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 846.258027] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 846.258027] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 846.258027] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 846.640808] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=1286,nova_cell1:SAVEPOINT=72,nova_cell1:RELEASE=72,nova_cell1:UPDATE=383,nova_cell1:INSERT=66,nova_cell1:DELETE=2 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 847.953290] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Expecting reply to msg e0dee2b9544246b09c7d940913799b45 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 848.123546] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg e0dee2b9544246b09c7d940913799b45 [ 848.124347] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Took 0.17 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 848.141082] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 848.141328] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 848.141499] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 848.203659] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 848.204108] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 848.204289] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 848.204652] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 848.205152] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 848.205275] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 848.215721] nova-conductor[53893]: DEBUG nova.quota [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Getting quotas for project e7e2429ac7b9497d86ed5fedc91c0dec. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 848.218105] nova-conductor[53893]: DEBUG nova.quota [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Getting quotas for user 9c1d1cb0f2d94cebb81e5e7b8f751827 and project e7e2429ac7b9497d86ed5fedc91c0dec. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 848.224177] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] [instance: 11943a7b-9afa-4e9c-84af-079ccc2c7cf1] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 848.224679] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 848.224939] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 848.225141] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 848.234404] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] [instance: 11943a7b-9afa-4e9c-84af-079ccc2c7cf1] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 848.235038] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 848.235253] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 848.235424] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 848.252700] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 848.252919] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 848.253122] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 849.195125] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] [instance: 1c342a36-b05d-452e-bbe1-fedf93e9f9d1] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port fece8e69-07e9-4015-ba5e-7aa4a9fb00ef, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 1c342a36-b05d-452e-bbe1-fedf93e9f9d1 was re-scheduled: Binding failed for port fece8e69-07e9-4015-ba5e-7aa4a9fb00ef, please check neutron logs for more information.\n'] [ 849.195758] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 849.196018] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1c342a36-b05d-452e-bbe1-fedf93e9f9d1.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1c342a36-b05d-452e-bbe1-fedf93e9f9d1. [ 849.196355] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] [instance: 1c342a36-b05d-452e-bbe1-fedf93e9f9d1] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1c342a36-b05d-452e-bbe1-fedf93e9f9d1. [ 849.234522] nova-conductor[53893]: DEBUG nova.network.neutron [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] [instance: 1c342a36-b05d-452e-bbe1-fedf93e9f9d1] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 849.299777] nova-conductor[53893]: DEBUG nova.network.neutron [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] [instance: 1c342a36-b05d-452e-bbe1-fedf93e9f9d1] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 849.303781] nova-conductor[53893]: DEBUG nova.network.neutron [None req-a27468d7-faa8-4a7c-9b09-7d2b4445245d tempest-AttachVolumeTestJSON-246792021 tempest-AttachVolumeTestJSON-246792021-project-member] [instance: 1c342a36-b05d-452e-bbe1-fedf93e9f9d1] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 850.671072] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Expecting reply to msg c33bd90f870f408d8f9a5dd321f8a23e in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 850.842715] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c33bd90f870f408d8f9a5dd321f8a23e [ 850.843088] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Took 0.18 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 850.866420] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 850.867388] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 850.867388] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 850.909944] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 850.909944] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 850.910221] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 850.910389] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 850.910601] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 850.910760] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 850.924138] nova-conductor[53893]: DEBUG nova.quota [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Getting quotas for project 8282d1df83f045fc9c91f25895b631de. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 850.925748] nova-conductor[53893]: DEBUG nova.quota [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Getting quotas for user 62706b3903c046f1b262416bcf373961 and project 8282d1df83f045fc9c91f25895b631de. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 850.932951] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] [instance: b56a9692-3745-4513-879c-4298716c5e81] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 850.933333] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 850.933541] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 850.933722] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 850.942670] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] [instance: b56a9692-3745-4513-879c-4298716c5e81] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 850.942670] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 850.942670] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 850.942670] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 850.972077] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 850.972318] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 850.972486] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 851.438469] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: b21f2e22-a6d6-4ab5-baa7-110004aa776e] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port bd06f62e-7763-4ae9-a899-977980921581, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance b21f2e22-a6d6-4ab5-baa7-110004aa776e was re-scheduled: Binding failed for port bd06f62e-7763-4ae9-a899-977980921581, please check neutron logs for more information.\n'] [ 851.439312] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 851.439488] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b21f2e22-a6d6-4ab5-baa7-110004aa776e.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b21f2e22-a6d6-4ab5-baa7-110004aa776e. [ 851.440133] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: b21f2e22-a6d6-4ab5-baa7-110004aa776e] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b21f2e22-a6d6-4ab5-baa7-110004aa776e. [ 851.494199] nova-conductor[53893]: DEBUG nova.network.neutron [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: b21f2e22-a6d6-4ab5-baa7-110004aa776e] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 851.592171] nova-conductor[53893]: DEBUG nova.network.neutron [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: b21f2e22-a6d6-4ab5-baa7-110004aa776e] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 851.605823] nova-conductor[53893]: DEBUG nova.network.neutron [None req-0ac45a8d-bf06-4806-ba46-b01bb5244a37 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: b21f2e22-a6d6-4ab5-baa7-110004aa776e] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 852.198437] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=1336,nova_cell1:UPDATE=388,nova_cell1:INSERT=45,nova_cell1:SAVEPOINT=65,nova_cell1:RELEASE=65,nova_cell1:DELETE=2 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 853.552179] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Expecting reply to msg c2aa86107859480587bddbbadd2d9854 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 853.582837] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] [instance: 5991598e-20cd-475d-bbdc-83cd4909a31e] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port bb82d309-55b2-42a9-9769-0594e64c862b, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 5991598e-20cd-475d-bbdc-83cd4909a31e was re-scheduled: Binding failed for port bb82d309-55b2-42a9-9769-0594e64c862b, please check neutron logs for more information.\n'] [ 853.585911] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 853.586218] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 5991598e-20cd-475d-bbdc-83cd4909a31e.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 5991598e-20cd-475d-bbdc-83cd4909a31e. [ 853.587291] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] [instance: 5991598e-20cd-475d-bbdc-83cd4909a31e] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 5991598e-20cd-475d-bbdc-83cd4909a31e. [ 853.621857] nova-conductor[53894]: DEBUG nova.network.neutron [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] [instance: 5991598e-20cd-475d-bbdc-83cd4909a31e] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 853.675016] nova-conductor[53894]: DEBUG nova.network.neutron [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] [instance: 5991598e-20cd-475d-bbdc-83cd4909a31e] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 853.682719] nova-conductor[53894]: DEBUG nova.network.neutron [None req-ed842645-5af2-49f1-9a07-bccc4a87cac5 tempest-ServerGroupTestJSON-928422351 tempest-ServerGroupTestJSON-928422351-project-member] [instance: 5991598e-20cd-475d-bbdc-83cd4909a31e] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 853.692009] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c2aa86107859480587bddbbadd2d9854 [ 853.694524] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Took 0.15 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 853.711484] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 853.711868] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 853.712194] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 853.759025] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 853.759025] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 853.759025] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 853.759025] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 853.759025] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 853.759025] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 853.768889] nova-conductor[53893]: DEBUG nova.quota [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Getting quotas for project 5a6790faf73648b89498c2305f6aea85. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 853.772551] nova-conductor[53893]: DEBUG nova.quota [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Getting quotas for user 2d301f3e5574407e9b13ce81a785466c and project 5a6790faf73648b89498c2305f6aea85. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 853.778029] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] [instance: b89e3f53-d9cd-400a-8ba9-83e328a59de8] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 853.778494] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 853.778696] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 853.778865] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 853.781993] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] [instance: b89e3f53-d9cd-400a-8ba9-83e328a59de8] block_device_mapping [BlockDeviceMapping(attachment_id=d2bd16d1-cf71-459c-9e40-5e5598e9c8f7,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='volume',device_name=None,device_type=None,disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id=None,instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='volume',tag=None,updated_at=,uuid=,volume_id='99ac65c2-3880-4653-9c73-253afc76a2f5',volume_size=1,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 853.782663] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 853.782882] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 853.783081] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 853.798891] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 853.799115] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 853.799286] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 854.987366] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_api:SELECT=212,nova_api:UPDATE=9,nova_api:DELETE=6 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 854.987871] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_api:SELECT=201,nova_api:DELETE=13,nova_api:UPDATE=10 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 854.996975] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Expecting reply to msg 95056843d77c4b54935df89315c75278 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 855.141849] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 95056843d77c4b54935df89315c75278 [ 855.142644] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Took 0.15 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 855.161114] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 855.161351] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 855.161522] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 855.163258] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=162,nova_cell1:INSERT=350,nova_cell1:SAVEPOINT=8,nova_cell1:RELEASE=8 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 855.197869] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 855.198097] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 855.198265] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 855.198621] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 855.198792] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 855.199453] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 855.202235] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell0:SELECT=37 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 855.209471] nova-conductor[53894]: DEBUG nova.quota [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Getting quotas for project bff3be1976444e58a2b7be93d47f50ef. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 855.211881] nova-conductor[53894]: DEBUG nova.quota [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Getting quotas for user 84f05a61e0f6499bb35c44d254226249 and project bff3be1976444e58a2b7be93d47f50ef. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 855.217461] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: ead7b401-1fd8-49b8-958a-2537714f22ca] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 855.217882] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 855.218130] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 855.218257] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 855.223296] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: ead7b401-1fd8-49b8-958a-2537714f22ca] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 855.223971] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 855.224182] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 855.224350] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 855.244833] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 855.245113] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 855.245293] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 855.817047] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 88a6495c-b9b7-4822-9cae-f27594545b3e] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port b0b09584-4bb2-4088-8468-c5351f0d451f, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 88a6495c-b9b7-4822-9cae-f27594545b3e was re-scheduled: Binding failed for port b0b09584-4bb2-4088-8468-c5351f0d451f, please check neutron logs for more information.\n'] [ 855.818221] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 855.818221] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 88a6495c-b9b7-4822-9cae-f27594545b3e.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 88a6495c-b9b7-4822-9cae-f27594545b3e. [ 855.818710] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 88a6495c-b9b7-4822-9cae-f27594545b3e] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 88a6495c-b9b7-4822-9cae-f27594545b3e. [ 855.851395] nova-conductor[53893]: DEBUG nova.network.neutron [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 88a6495c-b9b7-4822-9cae-f27594545b3e] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 855.909182] nova-conductor[53893]: DEBUG nova.network.neutron [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 88a6495c-b9b7-4822-9cae-f27594545b3e] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 855.913455] nova-conductor[53893]: DEBUG nova.network.neutron [None req-3352acb2-2519-4d95-a938-7905a23f9309 tempest-ServersTestJSON-1597439793 tempest-ServersTestJSON-1597439793-project-member] [instance: 88a6495c-b9b7-4822-9cae-f27594545b3e] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 858.349217] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] [instance: 11943a7b-9afa-4e9c-84af-079ccc2c7cf1] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 82010836-c561-4ba9-bf30-048d661c0450, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 11943a7b-9afa-4e9c-84af-079ccc2c7cf1 was re-scheduled: Binding failed for port 82010836-c561-4ba9-bf30-048d661c0450, please check neutron logs for more information.\n'] [ 858.350314] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 858.350397] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 11943a7b-9afa-4e9c-84af-079ccc2c7cf1.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 11943a7b-9afa-4e9c-84af-079ccc2c7cf1. [ 858.350741] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] [instance: 11943a7b-9afa-4e9c-84af-079ccc2c7cf1] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 11943a7b-9afa-4e9c-84af-079ccc2c7cf1. [ 858.379286] nova-conductor[53894]: DEBUG nova.network.neutron [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] [instance: 11943a7b-9afa-4e9c-84af-079ccc2c7cf1] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 858.443274] nova-conductor[53894]: DEBUG nova.network.neutron [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] [instance: 11943a7b-9afa-4e9c-84af-079ccc2c7cf1] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 858.452436] nova-conductor[53894]: DEBUG nova.network.neutron [None req-0b8d0c24-6031-4c9f-9d22-7e01f7ce2fea tempest-ServersTestJSON-1540490964 tempest-ServersTestJSON-1540490964-project-member] [instance: 11943a7b-9afa-4e9c-84af-079ccc2c7cf1] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 858.514441] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] [instance: 28bcec42-4fb0-4ef1-b882-6224fdbcec16] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn\n self._fetch_image_if_missing(context, vi)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing\n image_cache(vi, tmp_image_ds_loc)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image\n vm_util.copy_virtual_disk(\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk\n session._wait_for_task(vmdk_copy_task)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task\n return self.wait_for_task(task_ref)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task\n return evt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner\n self.f(*self.args, **self.kw)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task\n raise exceptions.translate_fault(task_info.error)\n', "oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n", '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', "nova.exception.RescheduledException: Build of instance 28bcec42-4fb0-4ef1-b882-6224fdbcec16 was re-scheduled: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n"] [ 858.515361] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 858.515600] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 28bcec42-4fb0-4ef1-b882-6224fdbcec16.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 28bcec42-4fb0-4ef1-b882-6224fdbcec16. [ 858.515829] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-498c7327-85e1-40cc-8f36-d708b0c38654 tempest-ServerShowV254Test-182182488 tempest-ServerShowV254Test-182182488-project-member] [instance: 28bcec42-4fb0-4ef1-b882-6224fdbcec16] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 28bcec42-4fb0-4ef1-b882-6224fdbcec16. [ 861.509417] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] [instance: b56a9692-3745-4513-879c-4298716c5e81] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 9cc9c772-a915-4d6d-8e40-aa6981387e9b, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance b56a9692-3745-4513-879c-4298716c5e81 was re-scheduled: Binding failed for port 9cc9c772-a915-4d6d-8e40-aa6981387e9b, please check neutron logs for more information.\n'] [ 861.510696] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 861.511263] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b56a9692-3745-4513-879c-4298716c5e81.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b56a9692-3745-4513-879c-4298716c5e81. [ 861.511569] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] [instance: b56a9692-3745-4513-879c-4298716c5e81] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b56a9692-3745-4513-879c-4298716c5e81. [ 861.545136] nova-conductor[53894]: DEBUG nova.network.neutron [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] [instance: b56a9692-3745-4513-879c-4298716c5e81] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 861.646249] nova-conductor[53894]: DEBUG nova.network.neutron [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] [instance: b56a9692-3745-4513-879c-4298716c5e81] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 861.650918] nova-conductor[53894]: DEBUG nova.network.neutron [None req-c738014b-17a0-49fd-8444-2689fa95efc3 tempest-ServerMetadataNegativeTestJSON-11169287 tempest-ServerMetadataNegativeTestJSON-11169287-project-member] [instance: b56a9692-3745-4513-879c-4298716c5e81] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 863.762249] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell0:SELECT=16 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 863.784980] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=72,nova_cell1:INSERT=156,nova_cell1:SAVEPOINT=4,nova_cell1:RELEASE=4 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 863.794810] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=96,nova_api:UPDATE=5,nova_api:DELETE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 863.796633] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=96,nova_api:DELETE=7,nova_api:UPDATE=3 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 864.144817] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Expecting reply to msg 4b1c8946909d452f8c944c9c544bc13f in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 864.305153] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 4b1c8946909d452f8c944c9c544bc13f [ 864.306306] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Took 0.17 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 864.333066] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: ead7b401-1fd8-49b8-958a-2537714f22ca] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port ea2e1bce-3212-4e30-9563-3347fadd7f9f, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance ead7b401-1fd8-49b8-958a-2537714f22ca was re-scheduled: Binding failed for port ea2e1bce-3212-4e30-9563-3347fadd7f9f, please check neutron logs for more information.\n'] [ 864.336195] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 864.336547] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ead7b401-1fd8-49b8-958a-2537714f22ca.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ead7b401-1fd8-49b8-958a-2537714f22ca. [ 864.336939] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: ead7b401-1fd8-49b8-958a-2537714f22ca] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance ead7b401-1fd8-49b8-958a-2537714f22ca. [ 864.382456] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 864.382643] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 864.382803] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 864.407634] nova-conductor[53894]: DEBUG nova.network.neutron [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: ead7b401-1fd8-49b8-958a-2537714f22ca] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 864.441598] nova-conductor[53894]: DEBUG nova.network.neutron [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: ead7b401-1fd8-49b8-958a-2537714f22ca] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 864.444865] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 864.445493] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 864.445493] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 864.446035] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 864.446035] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 864.446035] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 864.450591] nova-conductor[53894]: DEBUG nova.network.neutron [None req-37f51081-b997-4174-8587-c095e18d99b7 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: ead7b401-1fd8-49b8-958a-2537714f22ca] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 864.466832] nova-conductor[53894]: DEBUG nova.quota [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Getting quotas for project 51ed8d04e4d94b4289dedd35273775da. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 864.470927] nova-conductor[53894]: DEBUG nova.quota [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Getting quotas for user caaa86bf3e484a109bb48f077f96a95c and project 51ed8d04e4d94b4289dedd35273775da. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 864.478817] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] [instance: 0a85edf7-72d1-471b-b543-8a91bf585cba] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 864.479166] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 864.479412] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 864.479631] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 864.486397] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] [instance: 0a85edf7-72d1-471b-b543-8a91bf585cba] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 864.487179] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 864.487682] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 864.487682] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 864.505139] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] [instance: b89e3f53-d9cd-400a-8ba9-83e328a59de8] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port a230a46e-0d35-42de-8308-d37547cc8465, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance b89e3f53-d9cd-400a-8ba9-83e328a59de8 was re-scheduled: Binding failed for port a230a46e-0d35-42de-8308-d37547cc8465, please check neutron logs for more information.\n'] [ 864.505139] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 864.505139] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b89e3f53-d9cd-400a-8ba9-83e328a59de8.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b89e3f53-d9cd-400a-8ba9-83e328a59de8. [ 864.505139] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] [instance: b89e3f53-d9cd-400a-8ba9-83e328a59de8] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance b89e3f53-d9cd-400a-8ba9-83e328a59de8. [ 864.529108] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 864.529543] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 864.529543] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 864.553070] nova-conductor[53894]: DEBUG nova.network.neutron [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] [instance: b89e3f53-d9cd-400a-8ba9-83e328a59de8] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 864.576161] nova-conductor[53894]: DEBUG nova.network.neutron [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] [instance: b89e3f53-d9cd-400a-8ba9-83e328a59de8] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 864.577855] nova-conductor[53894]: DEBUG nova.network.neutron [None req-ae466a58-1a89-456a-964b-4ed2dd49919a tempest-ServersTestBootFromVolume-630008723 tempest-ServersTestBootFromVolume-630008723-project-member] [instance: b89e3f53-d9cd-400a-8ba9-83e328a59de8] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 866.344148] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Expecting reply to msg a6f2d93b7af14a38b27d83f7ed061104 in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 866.492710] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg a6f2d93b7af14a38b27d83f7ed061104 [ 866.493409] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Took 0.15 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 866.507795] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 866.508516] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 866.508516] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 866.545606] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 866.545828] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 866.546011] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 866.546420] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 866.546604] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 866.546778] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 866.556753] nova-conductor[53893]: DEBUG nova.quota [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Getting quotas for project bff3be1976444e58a2b7be93d47f50ef. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 866.560291] nova-conductor[53893]: DEBUG nova.quota [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Getting quotas for user 84f05a61e0f6499bb35c44d254226249 and project bff3be1976444e58a2b7be93d47f50ef. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 866.566196] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 98cb1c0a-6fba-4e56-9211-6581d3eee790] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 866.566614] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 866.566814] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 866.566978] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 866.571447] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 98cb1c0a-6fba-4e56-9211-6581d3eee790] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 866.572106] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 866.572307] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 866.572487] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 866.591875] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 866.592133] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 866.592314] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 871.507428] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Expecting reply to msg c221129d5065418fb090e3c1e228efcd in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 871.653376] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] [instance: 0a85edf7-72d1-471b-b543-8a91bf585cba] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port ba385b5e-d417-4b48-aa68-2c36ff17e231, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 0a85edf7-72d1-471b-b543-8a91bf585cba was re-scheduled: Binding failed for port ba385b5e-d417-4b48-aa68-2c36ff17e231, please check neutron logs for more information.\n'] [ 871.654231] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 871.654231] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 0a85edf7-72d1-471b-b543-8a91bf585cba.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 0a85edf7-72d1-471b-b543-8a91bf585cba. [ 871.654390] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] [instance: 0a85edf7-72d1-471b-b543-8a91bf585cba] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 0a85edf7-72d1-471b-b543-8a91bf585cba. [ 871.665824] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg c221129d5065418fb090e3c1e228efcd [ 871.669628] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Took 0.17 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 871.688792] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 871.689046] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 871.689229] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 871.698191] nova-conductor[53894]: DEBUG nova.network.neutron [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] [instance: 0a85edf7-72d1-471b-b543-8a91bf585cba] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 871.723789] nova-conductor[53894]: DEBUG nova.network.neutron [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] [instance: 0a85edf7-72d1-471b-b543-8a91bf585cba] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 871.738485] nova-conductor[53894]: DEBUG nova.network.neutron [None req-13371d3e-b164-4012-b292-aa3a66806598 tempest-AttachInterfacesUnderV243Test-1251072297 tempest-AttachInterfacesUnderV243Test-1251072297-project-member] [instance: 0a85edf7-72d1-471b-b543-8a91bf585cba] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 871.770584] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 871.770798] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 871.770972] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 871.771343] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 871.771525] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 871.771720] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 871.783493] nova-conductor[53894]: DEBUG nova.quota [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Getting quotas for project d11c0741810b4ebe8fd2839f38dcdf4d. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 871.786202] nova-conductor[53894]: DEBUG nova.quota [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Getting quotas for user 40821b3f2b084dcca46c911428a61ced and project d11c0741810b4ebe8fd2839f38dcdf4d. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 871.794453] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] [instance: 23641d1e-bbca-4887-95c6-5a6cac0ce6a1] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 871.794898] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 871.795113] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 871.795776] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 871.798029] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] [instance: 23641d1e-bbca-4887-95c6-5a6cac0ce6a1] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 871.798657] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 871.798853] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 871.799025] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 871.814825] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 871.815199] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 871.815391] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 872.300274] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 98cb1c0a-6fba-4e56-9211-6581d3eee790] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 753, in spawn\n vm_ref = self.build_virtual_machine(instance,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 275, in build_virtual_machine\n vif_infos = vmwarevif.get_vif_info(self._session,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vif.py", line 119, in get_vif_info\n for vif in network_info:\n', ' File "/opt/stack/nova/nova/network/model.py", line 612, in __iter__\n return self._sync_wrapper(fn, *args, **kwargs)\n', ' File "/opt/stack/nova/nova/network/model.py", line 603, in _sync_wrapper\n self.wait()\n', ' File "/opt/stack/nova/nova/network/model.py", line 635, in wait\n self[:] = self._gt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 224, in wait\n return self._exit_event.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/greenthread.py", line 264, in main\n result = function(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/utils.py", line 664, in context_wrapper\n return func(*args, **kwargs)\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2002, in _allocate_network_async\n raise e\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 1980, in _allocate_network_async\n nwinfo = self.network_api.allocate_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1229, in allocate_for_instance\n created_port_ids = self._update_ports_for_instance(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1365, in _update_ports_for_instance\n with excutils.save_and_reraise_exception():\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 227, in __exit__\n self.force_reraise()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_utils/excutils.py", line 200, in force_reraise\n raise self.value\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 1340, in _update_ports_for_instance\n updated_port = self._update_port(\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 585, in _update_port\n _ensure_no_port_binding_failure(port)\n', ' File "/opt/stack/nova/nova/network/neutron.py", line 294, in _ensure_no_port_binding_failure\n raise exception.PortBindingFailed(port_id=port[\'id\'])\n', 'nova.exception.PortBindingFailed: Binding failed for port 7a5102a3-39f6-4a6e-9f3a-634c590edb10, please check neutron logs for more information.\n', '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', 'nova.exception.RescheduledException: Build of instance 98cb1c0a-6fba-4e56-9211-6581d3eee790 was re-scheduled: Binding failed for port 7a5102a3-39f6-4a6e-9f3a-634c590edb10, please check neutron logs for more information.\n'] [ 872.300898] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 872.301141] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 98cb1c0a-6fba-4e56-9211-6581d3eee790.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 98cb1c0a-6fba-4e56-9211-6581d3eee790. [ 872.301448] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 98cb1c0a-6fba-4e56-9211-6581d3eee790] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 98cb1c0a-6fba-4e56-9211-6581d3eee790. [ 872.327449] nova-conductor[53894]: DEBUG nova.network.neutron [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 98cb1c0a-6fba-4e56-9211-6581d3eee790] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 872.352992] nova-conductor[53894]: DEBUG nova.network.neutron [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 98cb1c0a-6fba-4e56-9211-6581d3eee790] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 872.358922] nova-conductor[53894]: DEBUG nova.network.neutron [None req-498aebc2-92d3-492c-b157-18d89d0f6de4 tempest-DeleteServersTestJSON-215265594 tempest-DeleteServersTestJSON-215265594-project-member] [instance: 98cb1c0a-6fba-4e56-9211-6581d3eee790] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 876.551050] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell0:SELECT=2 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 876.575052] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=9,nova_cell1:INSERT=19 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 876.585465] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=15 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 876.587564] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=9,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 879.575150] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Expecting reply to msg af3df703491c416a9c1e402639cbc35c in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 879.846236] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg af3df703491c416a9c1e402639cbc35c [ 879.846236] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Took 0.27 seconds to select destinations for 3 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 879.864035] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 879.864035] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 879.864035] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 879.903105] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 879.903524] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 879.903844] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 879.934030] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 879.934030] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 879.934030] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 879.976697] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 879.976923] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 879.977176] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 879.977519] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 879.977619] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 879.977788] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 879.988667] nova-conductor[53893]: DEBUG nova.quota [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Getting quotas for project c837b52a6dc84bda829c0012a3537199. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 879.991161] nova-conductor[53893]: DEBUG nova.quota [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Getting quotas for user 474d41dd1c6f42a6bdfa525e522b2d38 and project c837b52a6dc84bda829c0012a3537199. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 879.997746] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: cbad69d3-7908-4f15-94c5-1b2c2f4e79f1] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 879.998241] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 879.998445] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 879.998610] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 880.003641] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: cbad69d3-7908-4f15-94c5-1b2c2f4e79f1] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 880.004312] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 880.004509] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 880.004860] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 880.020405] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 880.020942] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 880.020942] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 880.028997] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: 299afd65-10d4-4602-9a7e-b5d12e88a823] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 880.029364] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 880.029588] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 880.029714] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 880.039031] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: 299afd65-10d4-4602-9a7e-b5d12e88a823] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 880.039031] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 880.039031] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 880.039031] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 880.073477] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 880.073703] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 880.073875] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 880.089245] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: 1ac59594-4fbc-4a99-9e73-657185d4f218] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 880.090026] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 880.091028] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.001s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 880.091028] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 880.109652] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: 1ac59594-4fbc-4a99-9e73-657185d4f218] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 880.110307] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 880.110506] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 880.110680] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 880.167368] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 880.167582] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 880.167745] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 881.775845] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell0:SELECT=5 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 881.802092] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=26,nova_cell1:INSERT=59,nova_cell1:SAVEPOINT=2,nova_cell1:RELEASE=2 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 881.811301] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_api:SELECT=35,nova_api:DELETE=1,nova_api:UPDATE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 881.812982] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_api:SELECT=35,nova_api:UPDATE=2,nova_api:DELETE=2 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 882.023939] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Expecting reply to msg d7c439ee70d54e0f88e7d71b562ce763 in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 882.193325] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg d7c439ee70d54e0f88e7d71b562ce763 [ 882.194112] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Took 0.17 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 882.208410] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 882.208630] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 882.208799] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 882.248128] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 882.248128] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 882.248128] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 882.248128] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 882.248128] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 882.248128] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 882.254856] nova-conductor[53894]: DEBUG nova.quota [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Getting quotas for project 97b1a06e12d741dbb654564a7ec4bfc6. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 882.258494] nova-conductor[53894]: DEBUG nova.quota [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Getting quotas for user a820ad08ac294a1da60fe38fcaceccbb and project 97b1a06e12d741dbb654564a7ec4bfc6. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 882.265724] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] [instance: 3a9e32f2-4300-4b44-ae16-67792000eb08] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 882.266167] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 882.266357] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 882.266518] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 882.275515] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] [instance: 3a9e32f2-4300-4b44-ae16-67792000eb08] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 882.275761] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 882.275962] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 882.276140] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 882.291225] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 882.291425] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 882.291588] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 889.982719] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell0:SELECT=2 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 890.114180] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=23,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=58,nova_cell1:RELEASE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 890.162782] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=20,nova_api:UPDATE=2,nova_api:DELETE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 890.163643] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=20,nova_api:UPDATE=1,nova_api:DELETE=2 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 892.250630] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell0:SELECT=2 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 892.278720] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=9,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=20,nova_cell1:RELEASE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 892.289208] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_api:SELECT=11 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 892.289684] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_api:SELECT=13,nova_api:UPDATE=1,nova_api:DELETE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 895.161237] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=549,nova_cell1:UPDATE=141,nova_cell1:SAVEPOINT=36,nova_cell1:RELEASE=36,nova_cell1:INSERT=21 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 895.390922] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:UPDATE=135,nova_cell1:SELECT=483,nova_cell1:SAVEPOINT=30,nova_cell1:RELEASE=30,nova_cell1:INSERT=24,nova_cell1:DELETE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 908.358855] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] [instance: d364fc2e-89d7-4b2e-a510-19148a8f1a2b] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn\n self._fetch_image_if_missing(context, vi)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing\n image_cache(vi, tmp_image_ds_loc)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image\n vm_util.copy_virtual_disk(\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk\n session._wait_for_task(vmdk_copy_task)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task\n return self.wait_for_task(task_ref)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task\n return evt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner\n self.f(*self.args, **self.kw)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task\n raise exceptions.translate_fault(task_info.error)\n', "oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n", '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', "nova.exception.RescheduledException: Build of instance d364fc2e-89d7-4b2e-a510-19148a8f1a2b was re-scheduled: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n"] [ 908.359501] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 908.359788] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d364fc2e-89d7-4b2e-a510-19148a8f1a2b.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d364fc2e-89d7-4b2e-a510-19148a8f1a2b. [ 908.360133] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-2179efab-b9e1-4d16-862f-e0ad9e398a90 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] [instance: d364fc2e-89d7-4b2e-a510-19148a8f1a2b] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance d364fc2e-89d7-4b2e-a510-19148a8f1a2b. [ 918.366982] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=21,nova_cell1:UPDATE=7,nova_cell1:SAVEPOINT=1,nova_cell1:RELEASE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 918.388017] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=18,nova_cell1:SAVEPOINT=1,nova_cell1:RELEASE=1,nova_cell1:UPDATE=4,nova_cell1:INSERT=2 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 942.854764] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=8,nova_cell1:UPDATE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 944.891893] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=9,nova_cell1:UPDATE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 955.325046] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] [instance: dc31ebf5-889b-438b-9f54-6df807714a38] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn\n self._fetch_image_if_missing(context, vi)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing\n image_cache(vi, tmp_image_ds_loc)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image\n vm_util.copy_virtual_disk(\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk\n session._wait_for_task(vmdk_copy_task)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task\n return self.wait_for_task(task_ref)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task\n return evt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner\n self.f(*self.args, **self.kw)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task\n raise exceptions.translate_fault(task_info.error)\n', "oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n", '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', "nova.exception.RescheduledException: Build of instance dc31ebf5-889b-438b-9f54-6df807714a38 was re-scheduled: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n"] [ 955.325655] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 955.325759] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance dc31ebf5-889b-438b-9f54-6df807714a38.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance dc31ebf5-889b-438b-9f54-6df807714a38. [ 955.326015] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-5ffad98d-ee1e-4988-b3fe-4586f9d58332 tempest-ServerShowV247Test-65457029 tempest-ServerShowV247Test-65457029-project-member] [instance: dc31ebf5-889b-438b-9f54-6df807714a38] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance dc31ebf5-889b-438b-9f54-6df807714a38. [ 957.795991] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Expecting reply to msg 7afa622cf5484a7cbfb77b53444a6c5c in queue reply_a3b4a07fec7c47f79edd27bfc1b445f0 [ 957.977026] nova-conductor[53893]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg 7afa622cf5484a7cbfb77b53444a6c5c [ 957.977792] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Took 0.18 seconds to select destinations for 1 instance(s). {{(pid=53893) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 957.989925] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 957.990144] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 957.990315] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 958.027595] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 958.027832] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 958.028025] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 958.028389] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 958.028572] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 958.028728] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 958.036642] nova-conductor[53893]: DEBUG nova.quota [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Getting quotas for project a2b9019e04914800ae05f83ca5814555. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 958.038874] nova-conductor[53893]: DEBUG nova.quota [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Getting quotas for user a00c6dd4f2f1467dbd66d9c6a63e3a81 and project a2b9019e04914800ae05f83ca5814555. Resources: {'cores', 'instances', 'ram'} {{(pid=53893) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 958.044277] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] [instance: 6d514620-06f3-4bd9-8d74-3f11a064ef58] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53893) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 958.044678] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 958.044875] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 958.045050] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 958.049840] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] [instance: 6d514620-06f3-4bd9-8d74-3f11a064ef58] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53893) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 958.050455] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 958.050647] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 958.050807] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 958.066526] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 958.066715] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 958.066883] nova-conductor[53893]: DEBUG oslo_concurrency.lockutils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53893) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 968.033347] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell0:SELECT=2 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 968.053487] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=9,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=20,nova_cell1:RELEASE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 968.060919] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=11,nova_api:UPDATE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 968.065571] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_api:SELECT=13,nova_api:DELETE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 968.437251] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=43,nova_cell1:UPDATE=14,nova_cell1:SAVEPOINT=1,nova_cell1:RELEASE=1,nova_cell1:INSERT=2 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 968.469896] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=42,nova_cell1:SAVEPOINT=1,nova_cell1:RELEASE=1,nova_cell1:UPDATE=10,nova_cell1:INSERT=2 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 981.909931] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=2,nova_cell1:UPDATE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 981.910309] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=2,nova_cell1:UPDATE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1003.278248] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] [instance: 0f6368a9-cadc-46b4-be16-017724580876] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn\n self._fetch_image_if_missing(context, vi)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing\n image_cache(vi, tmp_image_ds_loc)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image\n vm_util.copy_virtual_disk(\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk\n session._wait_for_task(vmdk_copy_task)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task\n return self.wait_for_task(task_ref)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task\n return evt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner\n self.f(*self.args, **self.kw)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task\n raise exceptions.translate_fault(task_info.error)\n', "oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n", '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', "nova.exception.RescheduledException: Build of instance 0f6368a9-cadc-46b4-be16-017724580876 was re-scheduled: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n"] [ 1003.278805] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 1003.279036] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 0f6368a9-cadc-46b4-be16-017724580876.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 0f6368a9-cadc-46b4-be16-017724580876. [ 1003.279380] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-599e8cd4-097b-4df0-a473-f7cbf2a8b22e tempest-ServersAaction247Test-1005626433 tempest-ServersAaction247Test-1005626433-project-member] [instance: 0f6368a9-cadc-46b4-be16-017724580876] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 0f6368a9-cadc-46b4-be16-017724580876. [ 1013.287015] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=27,nova_cell1:UPDATE=8,nova_cell1:SAVEPOINT=1,nova_cell1:RELEASE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1013.305498] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=25,nova_cell1:SAVEPOINT=1,nova_cell1:RELEASE=1,nova_cell1:UPDATE=4,nova_cell1:INSERT=2 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1050.431228] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] [instance: 23641d1e-bbca-4887-95c6-5a6cac0ce6a1] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn\n self._fetch_image_if_missing(context, vi)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing\n image_cache(vi, tmp_image_ds_loc)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image\n vm_util.copy_virtual_disk(\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk\n session._wait_for_task(vmdk_copy_task)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task\n return self.wait_for_task(task_ref)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task\n return evt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner\n self.f(*self.args, **self.kw)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task\n raise exceptions.translate_fault(task_info.error)\n', "oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n", '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', "nova.exception.RescheduledException: Build of instance 23641d1e-bbca-4887-95c6-5a6cac0ce6a1 was re-scheduled: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n"] [ 1050.431228] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 1050.431582] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 23641d1e-bbca-4887-95c6-5a6cac0ce6a1.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 23641d1e-bbca-4887-95c6-5a6cac0ce6a1. [ 1050.431643] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] [instance: 23641d1e-bbca-4887-95c6-5a6cac0ce6a1] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 23641d1e-bbca-4887-95c6-5a6cac0ce6a1. [ 1050.452421] nova-conductor[53894]: DEBUG nova.network.neutron [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] [instance: 23641d1e-bbca-4887-95c6-5a6cac0ce6a1] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1050.520978] nova-conductor[53894]: DEBUG nova.network.neutron [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] [instance: 23641d1e-bbca-4887-95c6-5a6cac0ce6a1] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1050.524518] nova-conductor[53894]: DEBUG nova.network.neutron [None req-9236c549-cb11-4864-85fa-a2883f24c113 tempest-ServerRescueTestJSON-513893860 tempest-ServerRescueTestJSON-513893860-project-member] [instance: 23641d1e-bbca-4887-95c6-5a6cac0ce6a1] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1064.832827] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=26,nova_cell1:UPDATE=8,nova_cell1:INSERT=1,nova_cell1:SAVEPOINT=1,nova_cell1:RELEASE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1064.892855] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=28,nova_cell1:UPDATE=9,nova_cell1:INSERT=1,nova_cell1:SAVEPOINT=1,nova_cell1:RELEASE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1086.777067] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=4,nova_cell1:UPDATE=2,nova_cell1:INSERT=2 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1087.856477] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=4,nova_cell1:UPDATE=2,nova_cell1:INSERT=2 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1100.279186] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: 1ac59594-4fbc-4a99-9e73-657185d4f218] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn\n self._fetch_image_if_missing(context, vi)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing\n image_cache(vi, tmp_image_ds_loc)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image\n vm_util.copy_virtual_disk(\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk\n session._wait_for_task(vmdk_copy_task)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task\n return self.wait_for_task(task_ref)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task\n return evt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner\n self.f(*self.args, **self.kw)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task\n raise exceptions.translate_fault(task_info.error)\n', "oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n", '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', "nova.exception.RescheduledException: Build of instance 1ac59594-4fbc-4a99-9e73-657185d4f218 was re-scheduled: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n"] [ 1100.279762] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 1100.279999] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1ac59594-4fbc-4a99-9e73-657185d4f218.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1ac59594-4fbc-4a99-9e73-657185d4f218. [ 1100.280223] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: 1ac59594-4fbc-4a99-9e73-657185d4f218] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 1ac59594-4fbc-4a99-9e73-657185d4f218. [ 1100.301791] nova-conductor[53893]: DEBUG nova.network.neutron [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: 1ac59594-4fbc-4a99-9e73-657185d4f218] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1100.462577] nova-conductor[53893]: DEBUG nova.network.neutron [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: 1ac59594-4fbc-4a99-9e73-657185d4f218] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1100.466141] nova-conductor[53893]: DEBUG nova.network.neutron [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: 1ac59594-4fbc-4a99-9e73-657185d4f218] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1100.476600] nova-conductor[53893]: DEBUG nova.db.main.api [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Created instance_extra for 1ac59594-4fbc-4a99-9e73-657185d4f218 {{(pid=53893) instance_extra_update_by_uuid /opt/stack/nova/nova/db/main/api.py:2551}} [ 1110.440822] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=27,nova_cell1:UPDATE=20,nova_cell1:INSERT=1,nova_cell1:SAVEPOINT=2,nova_cell1:RELEASE=2,nova_cell1:DELETE=2 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1110.479077] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=30,nova_cell1:UPDATE=12,nova_cell1:INSERT=3,nova_cell1:SAVEPOINT=1,nova_cell1:ROLLBACK=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1125.808634] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=7 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1125.822098] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=6,nova_cell1:UPDATE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1150.789508] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: cbad69d3-7908-4f15-94c5-1b2c2f4e79f1] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn\n self._fetch_image_if_missing(context, vi)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing\n image_cache(vi, tmp_image_ds_loc)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image\n vm_util.copy_virtual_disk(\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk\n session._wait_for_task(vmdk_copy_task)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task\n return self.wait_for_task(task_ref)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task\n return evt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner\n self.f(*self.args, **self.kw)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task\n raise exceptions.translate_fault(task_info.error)\n', "oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n", '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', "nova.exception.RescheduledException: Build of instance cbad69d3-7908-4f15-94c5-1b2c2f4e79f1 was re-scheduled: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n"] [ 1150.790046] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 1150.790289] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance cbad69d3-7908-4f15-94c5-1b2c2f4e79f1.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance cbad69d3-7908-4f15-94c5-1b2c2f4e79f1. [ 1150.790498] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: cbad69d3-7908-4f15-94c5-1b2c2f4e79f1] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance cbad69d3-7908-4f15-94c5-1b2c2f4e79f1. [ 1150.813472] nova-conductor[53893]: DEBUG nova.network.neutron [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: cbad69d3-7908-4f15-94c5-1b2c2f4e79f1] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1150.831704] nova-conductor[53893]: DEBUG nova.network.neutron [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: cbad69d3-7908-4f15-94c5-1b2c2f4e79f1] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1150.834830] nova-conductor[53893]: DEBUG nova.network.neutron [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: cbad69d3-7908-4f15-94c5-1b2c2f4e79f1] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1160.974401] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=28,nova_cell1:UPDATE=12,nova_cell1:INSERT=1,nova_cell1:SAVEPOINT=1,nova_cell1:RELEASE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1163.998625] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=27,nova_cell1:UPDATE=20,nova_cell1:INSERT=2,nova_cell1:SAVEPOINT=2,nova_cell1:RELEASE=2,nova_cell1:DELETE=2 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1187.923051] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=13,nova_cell1:UPDATE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1187.936859] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=11,nova_cell1:DELETE=1,nova_cell1:UPDATE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1202.506708] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: 299afd65-10d4-4602-9a7e-b5d12e88a823] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn\n self._fetch_image_if_missing(context, vi)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing\n image_cache(vi, tmp_image_ds_loc)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image\n vm_util.copy_virtual_disk(\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk\n session._wait_for_task(vmdk_copy_task)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task\n return self.wait_for_task(task_ref)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task\n return evt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner\n self.f(*self.args, **self.kw)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task\n raise exceptions.translate_fault(task_info.error)\n', "oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n", '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', "nova.exception.RescheduledException: Build of instance 299afd65-10d4-4602-9a7e-b5d12e88a823 was re-scheduled: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n"] [ 1202.507331] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 1202.507574] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 299afd65-10d4-4602-9a7e-b5d12e88a823.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 299afd65-10d4-4602-9a7e-b5d12e88a823. [ 1202.507830] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: 299afd65-10d4-4602-9a7e-b5d12e88a823] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 299afd65-10d4-4602-9a7e-b5d12e88a823. [ 1202.531176] nova-conductor[53893]: DEBUG nova.network.neutron [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: 299afd65-10d4-4602-9a7e-b5d12e88a823] deallocate_for_instance() {{(pid=53893) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1202.553543] nova-conductor[53893]: DEBUG nova.network.neutron [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: 299afd65-10d4-4602-9a7e-b5d12e88a823] Instance cache missing network info. {{(pid=53893) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1202.557351] nova-conductor[53893]: DEBUG nova.network.neutron [None req-4917383e-606f-43c5-bef6-828c7aa83d6f tempest-ListServersNegativeTestJSON-267653004 tempest-ListServersNegativeTestJSON-267653004-project-member] [instance: 299afd65-10d4-4602-9a7e-b5d12e88a823] Updating instance_info_cache with network_info: [] {{(pid=53893) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1208.605399] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Expecting reply to msg b28b0f5d3799461d9129a51c31c214ad in queue reply_221f6b1d765f476aaa22fe0a28260c97 [ 1208.741731] nova-conductor[53894]: INFO oslo_messaging._drivers.amqpdriver [-] Received RPC response for msg b28b0f5d3799461d9129a51c31c214ad [ 1208.742542] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Took 0.14 seconds to select destinations for 1 instance(s). {{(pid=53894) _schedule_instances /opt/stack/nova/nova/conductor/manager.py:945}} [ 1208.757466] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1208.757466] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1208.757466] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1208.796502] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Acquiring lock "00000000-0000-0000-0000-000000000000" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1208.796732] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Lock "00000000-0000-0000-0000-000000000000" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1208.796901] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Lock "00000000-0000-0000-0000-000000000000" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1208.797270] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1208.797455] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1208.797616] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1208.805609] nova-conductor[53894]: DEBUG nova.quota [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Getting quotas for project ba125043b4bc4657b0fd2402613ebd93. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:393}} [ 1208.807898] nova-conductor[53894]: DEBUG nova.quota [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Getting quotas for user 13db58c4a3cb4f859e21d2c98e77afde and project ba125043b4bc4657b0fd2402613ebd93. Resources: {'cores', 'instances', 'ram'} {{(pid=53894) _get_quotas /opt/stack/nova/nova/quota.py:383}} [ 1208.813832] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] [instance: 0e943fcd-8c20-4835-9e43-f636ecf73366] Selected host: cpu-1; Selected node: domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28; Alternates: [] {{(pid=53894) schedule_and_build_instances /opt/stack/nova/nova/conductor/manager.py:1761}} [ 1208.813926] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1208.814115] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1208.814281] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1208.818731] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] [instance: 0e943fcd-8c20-4835-9e43-f636ecf73366] block_device_mapping [BlockDeviceMapping(attachment_id=,boot_index=0,connection_info=None,created_at=,delete_on_termination=True,deleted=,deleted_at=,destination_type='local',device_name=None,device_type='disk',disk_bus=None,encrypted=False,encryption_format=None,encryption_options=None,encryption_secret_uuid=None,guest_format=None,id=,image_id='9b91196f-102b-4380-9e69-c9f71c27118a',instance=,instance_uuid=,no_device=False,snapshot_id=None,source_type='image',tag=None,updated_at=,uuid=,volume_id=None,volume_size=None,volume_type=None)] {{(pid=53894) _create_block_device_mapping /opt/stack/nova/nova/conductor/manager.py:1506}} [ 1208.819317] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1208.819513] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1208.819679] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1208.832359] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Acquiring lock "bfcf983b-0431-4fe7-9afc-0efa54598587" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:402}} [ 1208.832555] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" acquired by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: waited 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:407}} [ 1208.832722] nova-conductor[53894]: DEBUG oslo_concurrency.lockutils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Lock "bfcf983b-0431-4fe7-9afc-0efa54598587" "released" by "nova.context.set_target_cell..get_or_set_cached_cell_and_set_connections" :: held 0.000s {{(pid=53894) inner /opt/stack/data/venv/lib/python3.10/site-packages/oslo_concurrency/lockutils.py:421}} [ 1218.802086] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell0:SELECT=2 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1218.822035] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=9,nova_cell1:SAVEPOINT=1,nova_cell1:INSERT=20,nova_cell1:RELEASE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1218.829223] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_api:SELECT=10,nova_api:UPDATE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1218.831801] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_api:SELECT=14,nova_api:DELETE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1222.326644] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=57,nova_cell1:UPDATE=20,nova_cell1:INSERT=4,nova_cell1:SAVEPOINT=2,nova_cell1:RELEASE=2 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1222.643024] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=57,nova_cell1:UPDATE=26,nova_cell1:INSERT=1,nova_cell1:SAVEPOINT=3,nova_cell1:RELEASE=3,nova_cell1:DELETE=2 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1247.787049] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=6,nova_cell1:UPDATE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1249.798564] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=7 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1250.504087] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] [instance: 3a9e32f2-4300-4b44-ae16-67792000eb08] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn\n self._fetch_image_if_missing(context, vi)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing\n image_cache(vi, tmp_image_ds_loc)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image\n vm_util.copy_virtual_disk(\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk\n session._wait_for_task(vmdk_copy_task)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task\n return self.wait_for_task(task_ref)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task\n return evt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner\n self.f(*self.args, **self.kw)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task\n raise exceptions.translate_fault(task_info.error)\n', "oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n", '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', "nova.exception.RescheduledException: Build of instance 3a9e32f2-4300-4b44-ae16-67792000eb08 was re-scheduled: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n"] [ 1250.504745] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 1250.505017] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 3a9e32f2-4300-4b44-ae16-67792000eb08.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 3a9e32f2-4300-4b44-ae16-67792000eb08. [ 1250.505365] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] [instance: 3a9e32f2-4300-4b44-ae16-67792000eb08] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 3a9e32f2-4300-4b44-ae16-67792000eb08. [ 1250.529504] nova-conductor[53894]: DEBUG nova.network.neutron [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] [instance: 3a9e32f2-4300-4b44-ae16-67792000eb08] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1250.550859] nova-conductor[53894]: DEBUG nova.network.neutron [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] [instance: 3a9e32f2-4300-4b44-ae16-67792000eb08] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1250.554295] nova-conductor[53894]: DEBUG nova.network.neutron [None req-3cd9b19d-caad-4c1c-a8ab-9a374674770e tempest-ServerActionsTestOtherA-627368268 tempest-ServerActionsTestOtherA-627368268-project-member] [instance: 3a9e32f2-4300-4b44-ae16-67792000eb08] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1260.690267] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=25,nova_cell1:UPDATE=19,nova_cell1:INSERT=1,nova_cell1:SAVEPOINT=2,nova_cell1:RELEASE=2,nova_cell1:DELETE=2 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1260.708339] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=28,nova_cell1:UPDATE=12,nova_cell1:INSERT=1,nova_cell1:SAVEPOINT=1,nova_cell1:RELEASE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1297.753246] nova-conductor[53893]: ERROR nova.scheduler.utils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] [instance: 6d514620-06f3-4bd9-8d74-3f11a064ef58] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn\n self._fetch_image_if_missing(context, vi)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing\n image_cache(vi, tmp_image_ds_loc)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image\n vm_util.copy_virtual_disk(\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk\n session._wait_for_task(vmdk_copy_task)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task\n return self.wait_for_task(task_ref)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task\n return evt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner\n self.f(*self.args, **self.kw)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task\n raise exceptions.translate_fault(task_info.error)\n', "oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n", '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', "nova.exception.RescheduledException: Build of instance 6d514620-06f3-4bd9-8d74-3f11a064ef58 was re-scheduled: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n"] [ 1297.753802] nova-conductor[53893]: DEBUG nova.conductor.manager [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Rescheduling: True {{(pid=53893) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 1297.754054] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 6d514620-06f3-4bd9-8d74-3f11a064ef58.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 6d514620-06f3-4bd9-8d74-3f11a064ef58. [ 1297.754287] nova-conductor[53893]: WARNING nova.scheduler.utils [None req-9d799411-0e05-48ed-a534-eaf776166255 tempest-ServerShowV257Test-578580744 tempest-ServerShowV257Test-578580744-project-member] [instance: 6d514620-06f3-4bd9-8d74-3f11a064ef58] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 6d514620-06f3-4bd9-8d74-3f11a064ef58. [ 1310.775777] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=39,nova_cell1:UPDATE=10,nova_cell1:SAVEPOINT=1,nova_cell1:RELEASE=1,nova_cell1:INSERT=2 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1311.550743] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=37,nova_cell1:UPDATE=19,nova_cell1:SAVEPOINT=3,nova_cell1:RELEASE=3,nova_cell1:DELETE=2 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1341.898440] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=2,nova_cell1:UPDATE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1341.925298] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=2 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1347.600025] nova-conductor[53894]: ERROR nova.scheduler.utils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] [instance: 0e943fcd-8c20-4835-9e43-f636ecf73366] Error from last host: cpu-1 (node domain-c8.fc996f14-c53b-4953-92e3-bdfa48f5cc28): ['Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2632, in _build_and_run_instance\n self.driver.spawn(context, instance, image_meta,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/driver.py", line 539, in spawn\n self._vmops.spawn(context, instance, image_meta, injected_files,\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 786, in spawn\n self._fetch_image_if_missing(context, vi)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 639, in _fetch_image_if_missing\n image_cache(vi, tmp_image_ds_loc)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vmops.py", line 537, in _cache_sparse_image\n vm_util.copy_virtual_disk(\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/vm_util.py", line 1423, in copy_virtual_disk\n session._wait_for_task(vmdk_copy_task)\n', ' File "/opt/stack/nova/nova/virt/vmwareapi/session.py", line 157, in _wait_for_task\n return self.wait_for_task(task_ref)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 398, in wait_for_task\n return evt.wait()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/event.py", line 124, in wait\n result = hub.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/eventlet/hubs/hub.py", line 310, in switch\n return self.greenlet.switch()\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/common/loopingcall.py", line 75, in _inner\n self.f(*self.args, **self.kw)\n', ' File "/opt/stack/data/venv/lib/python3.10/site-packages/oslo_vmware/api.py", line 448, in _poll_task\n raise exceptions.translate_fault(task_info.error)\n', "oslo_vmware.exceptions.VimFaultException: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n", '\nDuring handling of the above exception, another exception occurred:\n\n', 'Traceback (most recent call last):\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2447, in _do_build_and_run_instance\n self._build_and_run_instance(context, instance, image,\n', ' File "/opt/stack/nova/nova/compute/manager.py", line 2739, in _build_and_run_instance\n raise exception.RescheduledException(\n', "nova.exception.RescheduledException: Build of instance 0e943fcd-8c20-4835-9e43-f636ecf73366 was re-scheduled: A specified parameter was not correct: fileType\nFaults: ['InvalidArgument']\n"] [ 1347.600025] nova-conductor[53894]: DEBUG nova.conductor.manager [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Rescheduling: True {{(pid=53894) build_instances /opt/stack/nova/nova/conductor/manager.py:695}} [ 1347.600431] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] Failed to compute_task_build_instances: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 0e943fcd-8c20-4835-9e43-f636ecf73366.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 0e943fcd-8c20-4835-9e43-f636ecf73366. [ 1347.600431] nova-conductor[53894]: WARNING nova.scheduler.utils [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] [instance: 0e943fcd-8c20-4835-9e43-f636ecf73366] Setting instance to ERROR state.: nova.exception.MaxRetriesExceeded: Exceeded maximum number of retries. Exhausted all hosts available for retrying build failures for instance 0e943fcd-8c20-4835-9e43-f636ecf73366. [ 1347.622092] nova-conductor[53894]: DEBUG nova.network.neutron [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] [instance: 0e943fcd-8c20-4835-9e43-f636ecf73366] deallocate_for_instance() {{(pid=53894) deallocate_for_instance /opt/stack/nova/nova/network/neutron.py:1803}} [ 1347.641151] nova-conductor[53894]: DEBUG nova.network.neutron [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] [instance: 0e943fcd-8c20-4835-9e43-f636ecf73366] Instance cache missing network info. {{(pid=53894) _get_preexisting_port_ids /opt/stack/nova/nova/network/neutron.py:3323}} [ 1347.643994] nova-conductor[53894]: DEBUG nova.network.neutron [None req-0663d1cb-e614-410e-9db8-ea442706471b tempest-ServerTagsTestJSON-1574734888 tempest-ServerTagsTestJSON-1574734888-project-member] [instance: 0e943fcd-8c20-4835-9e43-f636ecf73366] Updating instance_info_cache with network_info: [] {{(pid=53894) update_instance_cache_with_nw_info /opt/stack/nova/nova/network/neutron.py:116}} [ 1357.608308] nova-conductor[53893]: DEBUG dbcounter [-] [53893] Writing DB stats nova_cell1:SELECT=19,nova_cell1:UPDATE=8,nova_cell1:INSERT=1,nova_cell1:SAVEPOINT=1,nova_cell1:RELEASE=1 {{(pid=53893) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1357.653629] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=18,nova_cell1:UPDATE=7,nova_cell1:INSERT=1,nova_cell1:SAVEPOINT=1,nova_cell1:RELEASE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}} [ 1370.739292] nova-conductor[53894]: DEBUG dbcounter [-] [53894] Writing DB stats nova_cell1:SELECT=6,nova_cell1:UPDATE=1 {{(pid=53894) stat_writer /opt/stack/data/venv/lib/python3.10/site-packages/dbcounter.py:115}}