xrt: refactor vk queue mutex into mutex per queue

Part-of: <https://gitlab.freedesktop.org/monado/monado/-/merge_requests/2594>

src/xrt/auxiliary/vk/vk_bundle_init.c

@@ -230,7 +230,7 @@ vk_fill_in_has_instance_extensions(struct vk_bundle *vk, struct u_string_list *e
  */
 
 static void
-fill_in_device_features(struct vk_bundle *vk)
+fill_in_device_features(struct vk_bundle *vk, const uint32_t queue_family_index)
 {
 	/*
 	 * Device properties.
@@ -252,12 +252,12 @@ fill_in_device_features(struct vk_bundle *vk)
 	uint32_t count = 0;
 	vk->vkGetPhysicalDeviceQueueFamilyProperties(vk->physical_device, &count, NULL);
 	assert(count != 0);
-	assert(count > vk->main_queue.family_index);
+	assert(count > queue_family_index);
 
 	VkQueueFamilyProperties *props = U_TYPED_ARRAY_CALLOC(VkQueueFamilyProperties, count);
 	vk->vkGetPhysicalDeviceQueueFamilyProperties(vk->physical_device, &count, props);
 
-	vk->features.timestamp_valid_bits = props[vk->main_queue.family_index].timestampValidBits;
+	vk->features.timestamp_valid_bits = props[queue_family_index].timestampValidBits;
 	free(props);
 }
 
@@ -1237,6 +1237,60 @@ filter_device_features(struct vk_bundle *vk,
 	         device_features->video_maintenance_1);                      //
 }
 
+static inline void
+vk_reset_queues(struct vk_bundle *vk)
+{
+	const struct vk_queue_pair null_queue_pair = VK_NULL_QUEUE_PAIR;
+
+	for (uint32_t i = 0; i < ARRAY_SIZE(vk->queues); ++i) {
+		struct vk_bundle_queue *q = &vk->queues[i];
+		q->queue = VK_NULL_HANDLE;
+		q->index = null_queue_pair.index;
+		q->family_index = null_queue_pair.family_index;
+	}
+}
+
+static struct vk_bundle_queue *
+vk_insert_get_queue(struct vk_bundle *vk, const struct vk_queue_pair *new_queue)
+{
+	if (vk == NULL ||                       //
+	    new_queue == NULL ||                //
+	    new_queue->index == (uint32_t)-1 || //
+	    new_queue->family_index == VK_QUEUE_FAMILY_IGNORED) {
+		return NULL;
+	}
+
+	/*
+	 * The same queue can be used for different uses, but we need to have
+	 * one mutex per queue, so look up if the queue has already been added.
+	 */
+	for (uint32_t i = 0; i < VK_BUNDLE_MAX_QUEUES; ++i) {
+		struct vk_bundle_queue *q = &vk->queues[i];
+		if (q->queue != VK_NULL_HANDLE &&   //
+		    new_queue->index == q->index && //
+		    new_queue->family_index == q->family_index) {
+			return q;
+		}
+	}
+
+	// Find the first unused queue bundle.
+	for (uint32_t i = 0; i < VK_BUNDLE_MAX_QUEUES; ++i) {
+		struct vk_bundle_queue *q = &vk->queues[i];
+		if (q->queue != VK_NULL_HANDLE) {
+			continue;
+		}
+
+		q->index = new_queue->index;
+		q->family_index = new_queue->family_index;
+
+		vk->vkGetDeviceQueue(vk->device, q->family_index, q->index, &q->queue);
+		return q;
+	}
+
+	VK_ERROR(vk, "failed to find a free queue, max queues reached");
+	return NULL;
+}
+
 
 /*
  *
@@ -1296,11 +1350,13 @@ vk_create_device(struct vk_bundle *vk,
 		return VK_ERROR_NOT_PERMITTED_EXT;
 	}
 
-	vk->main_queue = VK_BUNDLE_NULL_QUEUE;
+	vk_reset_queues(vk);
+
+	struct vk_queue_pair main_queue = VK_NULL_QUEUE_PAIR;
 	if (only_compute) {
-		ret = find_queue_family(vk, VK_QUEUE_COMPUTE_BIT, &vk->main_queue.family_index);
+		ret = find_queue_family(vk, VK_QUEUE_COMPUTE_BIT, &main_queue.family_index);
 	} else {
-		ret = find_graphics_queue_family(vk, &vk->main_queue.family_index);
+		ret = find_graphics_queue_family(vk, &main_queue.family_index);
 	}
 
 	if (ret != VK_SUCCESS) {
@@ -1318,27 +1374,27 @@ vk_create_device(struct vk_bundle *vk,
 	uint32_t queue_create_info_count = 1;
 
 	// Compute or Graphics queue
-	vk->main_queue.index = 0;
+	main_queue.index = 0;
 	queue_create_info[0].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO;
 	queue_create_info[0].pNext = NULL;
 	queue_create_info[0].queueCount = 1;
-	queue_create_info[0].queueFamilyIndex = vk->main_queue.family_index;
+	queue_create_info[0].queueFamilyIndex = main_queue.family_index;
 	queue_create_info[0].pQueuePriorities = &queue_priority;
 
 #ifdef VK_KHR_video_encode_queue
 	// Video encode queue
-	vk->encode_queue = VK_BUNDLE_NULL_QUEUE;
+	struct vk_queue_pair encode_queue = VK_NULL_QUEUE_PAIR;
 	if (u_string_list_contains(device_ext_list, VK_KHR_VIDEO_ENCODE_QUEUE_EXTENSION_NAME)) {
-		ret = find_queue_family(vk, VK_QUEUE_VIDEO_ENCODE_BIT_KHR, &vk->encode_queue.family_index);
+		ret = find_queue_family(vk, VK_QUEUE_VIDEO_ENCODE_BIT_KHR, &encode_queue.family_index);
 		if (ret == VK_SUCCESS) {
-			vk->encode_queue.index = 0;
+			encode_queue.index = 0;
 			queue_create_info[queue_create_info_count].sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO;
 			queue_create_info[queue_create_info_count].pNext = NULL;
 			queue_create_info[queue_create_info_count].queueCount = 1;
-			queue_create_info[queue_create_info_count].queueFamilyIndex = vk->encode_queue.family_index;
+			queue_create_info[queue_create_info_count].queueFamilyIndex = encode_queue.family_index;
 			queue_create_info[queue_create_info_count].pQueuePriorities = &queue_priority;
 			queue_create_info_count++;
-			VK_DEBUG(vk, "Creating video encode queue, family index %d", vk->encode_queue.family_index);
+			VK_DEBUG(vk, "Creating video encode queue, family index %d", encode_queue.family_index);
 		}
 	}
 #endif
@@ -1503,7 +1559,7 @@ vk_create_device(struct vk_bundle *vk,
 	}
 
 	// Fill in the device features we are interested in.
-	fill_in_device_features(vk);
+	fill_in_device_features(vk, main_queue.family_index);
 
 	// We fill in these here as we want to be sure we have selected the physical device fully.
 	fill_in_external_object_properties(vk);
@@ -1513,12 +1569,11 @@ vk_create_device(struct vk_bundle *vk,
 	if (ret != VK_SUCCESS) {
 		goto err_destroy;
 	}
-	vk->vkGetDeviceQueue(vk->device, vk->main_queue.family_index, vk->main_queue.index, &vk->main_queue.queue);
+
+	vk->main_queue = vk_insert_get_queue(vk, &main_queue);
+	assert(vk->main_queue != NULL);
 #if defined(VK_KHR_video_encode_queue)
-	if (vk->encode_queue.family_index != VK_QUEUE_FAMILY_IGNORED) {
-		vk->vkGetDeviceQueue(vk->device, vk->encode_queue.family_index, vk->encode_queue.index,
-		                     &vk->encode_queue.queue);
-	}
+	vk->encode_queue = vk_insert_get_queue(vk, &encode_queue);
 #endif
 
 	// Need to do this after functions have been gotten.
@@ -1537,8 +1592,15 @@ err_destroy:
 VkResult
 vk_init_mutex(struct vk_bundle *vk)
 {
-	if (os_mutex_init(&vk->queue_mutex) < 0) {
-		return VK_ERROR_INITIALIZATION_FAILED;
+	assert(vk != NULL);
+
+	for (uint32_t i = 0; i < ARRAY_SIZE(vk->queues); ++i) {
+		struct vk_bundle_queue *q = &vk->queues[i];
+
+		if (os_mutex_init(&q->mutex) < 0) {
+			VK_ERROR(vk, "failed to initialize queue mutex");
+			return VK_ERROR_INITIALIZATION_FAILED;
+		}
 	}
 	return VK_SUCCESS;
 }
@@ -1546,11 +1608,16 @@ vk_init_mutex(struct vk_bundle *vk)
 VkResult
 vk_deinit_mutex(struct vk_bundle *vk)
 {
-	os_mutex_destroy(&vk->queue_mutex);
+	assert(vk != NULL);
+
+	for (uint32_t i = 0; i < ARRAY_SIZE(vk->queues); ++i) {
+		struct vk_bundle_queue *q = &vk->queues[i];
+		os_mutex_destroy(&q->mutex);
+		q->mutex = (struct os_mutex){0};
+	}
 	return VK_SUCCESS;
 }
 
-
 /*
  *
  * Complete setup.
@@ -1578,6 +1645,8 @@ vk_init_from_given(struct vk_bundle *vk,
 	U_ZERO(vk);
 	vk->log_level = log_level;
 
+	vk_reset_queues(vk);
+
 	ret = vk_get_loader_functions(vk, vkGetInstanceProcAddr);
 	if (ret != VK_SUCCESS) {
 		goto err_memset;
@@ -1586,11 +1655,6 @@ vk_init_from_given(struct vk_bundle *vk,
 	vk->instance = instance;
 	vk->physical_device = physical_device;
 	vk->device = device;
-	vk->main_queue.family_index = queue_family_index;
-	vk->main_queue.index = queue_index;
-#ifdef VK_KHR_video_encode_queue
-	vk->encode_queue = VK_BUNDLE_NULL_QUEUE;
-#endif
 
 	// Fill in all instance functions.
 	ret = vk_get_instance_functions(vk);
@@ -1628,8 +1692,12 @@ vk_init_from_given(struct vk_bundle *vk,
 	}
 #endif
 
+	const struct vk_queue_pair main_queue = {
+	    .family_index = queue_family_index,
+	    .index = queue_index,
+	};
 	// Fill in the device features we are interested in.
-	fill_in_device_features(vk);
+	fill_in_device_features(vk, main_queue.family_index);
 
 	// Fill in external object properties.
 	fill_in_external_object_properties(vk);
@@ -1640,7 +1708,8 @@ vk_init_from_given(struct vk_bundle *vk,
 		goto err_memset;
 	}
 
-	vk->vkGetDeviceQueue(vk->device, vk->main_queue.family_index, vk->main_queue.index, &vk->main_queue.queue);
+	vk->main_queue = vk_insert_get_queue(vk, &main_queue);
+	assert(vk->main_queue != NULL);
 
 	vk->has_EXT_debug_utils = false;
 	if (debug_utils_enabled) {

src/xrt/auxiliary/vk/vk_cmd.c

@@ -83,9 +83,9 @@ vk_cmd_submit_locked(
 {
 	VkResult ret;
 
-	os_mutex_lock(&vk->queue_mutex);
+	vk_queue_lock(queue);
 	ret = vk->vkQueueSubmit(queue->queue, count, infos, fence);
-	os_mutex_unlock(&vk->queue_mutex);
+	vk_queue_unlock(queue);
 
 	if (ret != VK_SUCCESS) {
 		VK_ERROR(vk, "vkQueueSubmit: %s", vk_result_string(ret));

src/xrt/auxiliary/vk/vk_cmd_pool.h

@@ -72,7 +72,7 @@ vk_cmd_pool_init_for_queue(struct vk_bundle *vk,
 static inline XRT_CHECK_RESULT VkResult
 vk_cmd_pool_init(struct vk_bundle *vk, struct vk_cmd_pool *pool, VkCommandPoolCreateFlags flags)
 {
-	return vk_cmd_pool_init_for_queue(vk, pool, flags, &vk->main_queue);
+	return vk_cmd_pool_init_for_queue(vk, pool, flags, vk->main_queue);
 }
 
 /*!

src/xrt/auxiliary/vk/vk_helpers.h

@@ -33,6 +33,7 @@
 extern "C" {
 #endif
 
+#define VK_BUNDLE_MAX_QUEUES 2
 
 /*
  *
@@ -40,6 +41,20 @@ extern "C" {
  *
  */
 
+struct vk_queue_pair
+{
+	//! The queue family index
+	uint32_t family_index;
+	//! The queue (instance) index
+	uint32_t index;
+};
+
+#define VK_NULL_QUEUE_PAIR                                                                                             \
+	XRT_C11_COMPOUND(struct vk_queue_pair)                                                                         \
+	{                                                                                                              \
+		.family_index = VK_QUEUE_FAMILY_IGNORED, .index = (uint32_t)-1,                                        \
+	}
+
 struct vk_bundle_queue
 {
 	//! The Vulkan queue handle
@@ -48,14 +63,10 @@ struct vk_bundle_queue
 	uint32_t family_index;
 	//! The queue (instance) index
 	uint32_t index;
+	//! The queue mutex - @see vk_queue_lock, vk_queue_unlock
+	struct os_mutex mutex;
 };
 
-#define VK_BUNDLE_NULL_QUEUE                                                                                           \
-	XRT_C11_COMPOUND(struct vk_bundle_queue)                                                                       \
-	{                                                                                                              \
-		.queue = VK_NULL_HANDLE, .family_index = VK_QUEUE_FAMILY_IGNORED, .index = (uint32_t)-1,               \
-	}
-
 /*!
  * A bundle of Vulkan functions and objects, used by both @ref comp and @ref
  * comp_client. Note that they both have different instances of the object, and
@@ -72,12 +83,21 @@ struct vk_bundle
 	VkPhysicalDevice physical_device;
 	int physical_device_index;
 	VkDevice device;
-	struct vk_bundle_queue main_queue;
-#if defined(VK_KHR_video_encode_queue)
-	struct vk_bundle_queue encode_queue;
-#endif
 
-	struct os_mutex queue_mutex;
+	/*!
+	 * @brief queues - a free list of **unique** queues
+	 *
+	 * One per uniquely identifiable vk queue (family x instance index),
+	 * duplicate entries must not be stored.
+	 *
+	 * Should not be used directly, @see main_queue, encode_queue
+	 */
+	struct vk_bundle_queue queues[VK_BUNDLE_MAX_QUEUES];
+
+	struct vk_bundle_queue *main_queue;
+#if defined(VK_KHR_video_encode_queue)
+	struct vk_bundle_queue *encode_queue;
+#endif
 
 	struct
 	{
@@ -1087,6 +1107,27 @@ vk_init_mutex(struct vk_bundle *vk);
 VkResult
 vk_deinit_mutex(struct vk_bundle *vk);
 
+static inline void
+vk_queue_lock(struct vk_bundle_queue *q)
+{
+	assert(q != NULL);
+	os_mutex_lock(&q->mutex);
+}
+
+static inline int
+vk_queue_trylock(struct vk_bundle_queue *q)
+{
+	assert(q != NULL);
+	return os_mutex_trylock(&q->mutex);
+}
+
+static inline void
+vk_queue_unlock(struct vk_bundle_queue *q)
+{
+	assert(q != NULL);
+	os_mutex_unlock(&q->mutex);
+}
+
 /*!
  * Initialize a bundle with objects given to us by client code,
  * used by @ref client_vk_compositor in @ref comp_client.

src/xrt/auxiliary/vk/vk_print.c

@@ -39,13 +39,31 @@
 	} while (false)
 
 static inline void
-print_queue(u_pp_delegate_t dg, const char *prefix, const struct vk_bundle_queue *queue)
+print_queue_non_null(u_pp_delegate_t dg, const char *prefix, const struct vk_bundle_queue *queue)
 {
+	assert(queue != NULL);
 	PNT("%squeue.queue: %p", prefix, (const void *)queue->queue);
 	PNT("%squeue.index: %d", prefix, (int32_t)queue->index);
 	PNT("%squeue.family_index: %d", prefix, (int32_t)queue->family_index);
 }
 
+static inline void
+print_queue(u_pp_delegate_t dg, const char *prefix, const struct vk_bundle_queue *queue)
+{
+	if (queue != NULL) {
+		print_queue_non_null(dg, prefix, queue);
+	} else {
+		const struct vk_queue_pair null_queue_pair = VK_NULL_QUEUE_PAIR;
+
+		struct vk_bundle_queue null_queue = {
+		    .queue = VK_NULL_HANDLE,
+		    .family_index = null_queue_pair.family_index,
+		    .index = null_queue_pair.index,
+		};
+		print_queue_non_null(dg, prefix, &null_queue);
+	}
+}
+
 /*
  *
  * 'Exported' functions.
@@ -288,17 +306,14 @@ vk_print_display_surface_create_info(struct vk_bundle *vk,
 void
 vk_print_queues_info(const struct vk_bundle *vk, enum u_logging_level log_level)
 {
-	struct vk_bundle_queue encode_queue = VK_BUNDLE_NULL_QUEUE;
-#if defined(VK_KHR_video_encode_queue)
-	encode_queue = vk->encode_queue;
-#endif
-
 	struct u_pp_sink_stack_only sink;
 	u_pp_delegate_t dg = u_pp_sink_stack_only_init(&sink);
 
 	P("Selected Queues/Families:");
-	print_queue(dg, "main_", &vk->main_queue);
-	print_queue(dg, "encode_", &encode_queue);
+	print_queue(dg, "main_", vk->main_queue);
+#if defined(VK_KHR_video_encode_queue)
+	print_queue(dg, "encode_", vk->encode_queue);
+#endif
 
 	U_LOG_IFL(log_level, vk->log_level, "%s", sink.buffer);
 }

src/xrt/auxiliary/vk/vk_sync_objects.c

@@ -142,15 +142,15 @@ vk_create_and_submit_fence_native(struct vk_bundle *vk, xrt_graphics_sync_handle
 	 * Submit fence.
 	 */
 
-	os_mutex_lock(&vk->queue_mutex);
+	vk_queue_lock(vk->main_queue);
 
-	ret = vk->vkQueueSubmit(  //
-	    vk->main_queue.queue, // queue
-	    0,                    // submitCount
-	    NULL,                 // pSubmits
-	    fence);               // fence
+	ret = vk->vkQueueSubmit(   //
+	    vk->main_queue->queue, // queue
+	    0,                     // submitCount
+	    NULL,                  // pSubmits
+	    fence);                // fence
 
-	os_mutex_unlock(&vk->queue_mutex);
+	vk_queue_unlock(vk->main_queue);
 
 	if (ret != VK_SUCCESS) {
 		VK_ERROR(vk, "vkQueueSubmit: %s", vk_result_string(ret));

src/xrt/compositor/client/comp_vk_client.c

@@ -178,11 +178,11 @@ submit_semaphore(struct client_vk_compositor *c, xrt_result_t *out_xret)
 	    .pSignalSemaphores = semaphores,
 	};
 
-	ret = vk->vkQueueSubmit(  //
-	    vk->main_queue.queue, // queue
-	    1,                    // submitCount
-	    &submit_info,         // pSubmits
-	    VK_NULL_HANDLE);      // fence
+	ret = vk->vkQueueSubmit(   //
+	    vk->main_queue->queue, // queue
+	    1,                     // submitCount
+	    &submit_info,          // pSubmits
+	    VK_NULL_HANDLE);       // fence
 	if (ret != VK_SUCCESS) {
 		VK_ERROR(vk, "vkQueueSubmit: %s", vk_result_string(ret));
 		*out_xret = XRT_ERROR_VULKAN;
@@ -245,9 +245,9 @@ submit_fallback(struct client_vk_compositor *c, xrt_result_t *out_xret)
 		COMP_TRACE_IDENT(device_wait_idle);
 
 		// Last course of action fallback.
-		os_mutex_lock(&vk->queue_mutex);
-		vk->vkQueueWaitIdle(vk->main_queue.queue);
-		os_mutex_unlock(&vk->queue_mutex);
+		vk_queue_lock(vk->main_queue);
+		vk->vkQueueWaitIdle(vk->main_queue->queue);
+		vk_queue_unlock(vk->main_queue);
 	}
 
 	*out_xret = xrt_comp_layer_commit(&c->xcn->base, XRT_GRAPHICS_SYNC_HANDLE_INVALID);
@@ -738,7 +738,7 @@ client_vk_swapchain_create(struct xrt_compositor *xc,
 		    .dstAccessMask = 0,
 		    .oldLayout = barrier_optimal_layout,
 		    .newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
-		    .srcQueueFamilyIndex = vk->main_queue.family_index,
+		    .srcQueueFamilyIndex = vk->main_queue->family_index,
 		    .dstQueueFamilyIndex = VK_QUEUE_FAMILY_EXTERNAL,
 		    .image = sc->base.images[i],
 		    .subresourceRange = subresource_range,

src/xrt/compositor/main/comp_renderer.c

@@ -159,9 +159,9 @@ renderer_wait_queue_idle(struct comp_renderer *r)
 	COMP_TRACE_MARKER();
 	struct vk_bundle *vk = &r->c->base.vk;
 
-	os_mutex_lock(&vk->queue_mutex);
-	vk->vkQueueWaitIdle(vk->main_queue.queue);
-	os_mutex_unlock(&vk->queue_mutex);
+	vk_queue_lock(vk->main_queue);
+	vk->vkQueueWaitIdle(vk->main_queue->queue);
+	vk_queue_unlock(vk->main_queue);
 }
 
 static void
@@ -665,7 +665,7 @@ renderer_submit_queue(struct comp_renderer *r, VkCommandBuffer cmd, VkPipelineSt
 	 * us avoid taking a lot of locks. The queue lock will be taken by
 	 * @ref vk_cmd_submit_locked tho.
 	 */
-	ret = vk_cmd_submit_locked(vk, &vk->main_queue, 1, &comp_submit_info, r->fences[r->acquired_buffer]);
+	ret = vk_cmd_submit_locked(vk, vk->main_queue, 1, &comp_submit_info, r->fences[r->acquired_buffer]);
 
 	// We have now completed the submit, even if we failed.
 	comp_target_mark_submit_end(ct, frame_id, os_monotonic_get_ns());
@@ -741,13 +741,13 @@ renderer_present_swapchain_image(struct comp_renderer *r, uint64_t desired_prese
 	assert(!comp_frame_is_invalid_locked(&r->c->frame.rendering));
 	uint64_t render_complete_signal_value = (uint64_t)r->c->frame.rendering.id;
 
-	ret = comp_target_present(          //
-	    r->c->target,                   //
-	    r->c->base.vk.main_queue.queue, //
-	    r->acquired_buffer,             //
-	    render_complete_signal_value,   //
-	    desired_present_time_ns,        //
-	    present_slop_ns);               //
+	ret = comp_target_present(           //
+	    r->c->target,                    //
+	    r->c->base.vk.main_queue->queue, //
+	    r->acquired_buffer,              //
+	    render_complete_signal_value,    //
+	    desired_present_time_ns,         //
+	    present_slop_ns);                //
 	r->acquired_buffer = -1;
 
 	if (ret == VK_ERROR_OUT_OF_DATE_KHR || ret == VK_SUBOPTIMAL_KHR) {

src/xrt/compositor/main/comp_target_swapchain.c

@@ -682,7 +682,7 @@ comp_target_swapchain_create_images(struct comp_target *ct, const struct comp_ta
 	// Can we create swapchains from the surface on this device and queue.
 	ret = vk->vkGetPhysicalDeviceSurfaceSupportKHR( //
 	    vk->physical_device,                        // physicalDevice
-	    vk->main_queue.family_index,                // queueFamilyIndex
+	    vk->main_queue->family_index,               // queueFamilyIndex
 	    cts->surface.handle,                        // surface
 	    &supported);                                // pSupported
 	if (ret != VK_SUCCESS) {
@@ -923,9 +923,9 @@ comp_target_swapchain_present(struct comp_target *ct,
 
 
 	// Need to take the queue lock for present.
-	os_mutex_lock(&vk->queue_mutex);
+	vk_queue_lock(vk->main_queue);
 	VkResult ret = vk->vkQueuePresentKHR(queue, &present_info);
-	os_mutex_unlock(&vk->queue_mutex);
+	vk_queue_unlock(vk->main_queue);
 
 
 #ifdef VK_EXT_display_control

src/xrt/compositor/main/comp_window_peek.c

@@ -266,9 +266,9 @@ comp_window_peek_destroy(struct comp_window_peek **w_ptr)
 
 	struct vk_bundle *vk = get_vk(w);
 
-	os_mutex_lock(&vk->queue_mutex);
+	vk_queue_lock(vk->main_queue);
 	vk->vkDeviceWaitIdle(vk->device);
-	os_mutex_unlock(&vk->queue_mutex);
+	vk_queue_unlock(vk->main_queue);
 
 	vk_cmd_pool_lock(&w->pool);
 	vk->vkFreeCommandBuffers(vk->device, w->pool.pool, 1, &w->cmd);
@@ -434,7 +434,7 @@ comp_window_peek_blit(struct comp_window_peek *w, VkImage src, int32_t width, in
 	};
 
 	// Done writing commands, submit to queue.
-	ret = vk_cmd_submit_locked(vk, &vk->main_queue, 1, &submit, VK_NULL_HANDLE);
+	ret = vk_cmd_submit_locked(vk, vk->main_queue, 1, &submit, VK_NULL_HANDLE);
 
 	// Done submitting commands, unlock pool.
 	vk_cmd_pool_unlock(&w->pool);
@@ -456,9 +456,9 @@ comp_window_peek_blit(struct comp_window_peek *w, VkImage src, int32_t width, in
 	    .pResults = NULL,
 	};
 
-	os_mutex_lock(&vk->queue_mutex);
-	ret = vk->vkQueuePresentKHR(vk->main_queue.queue, &present);
-	os_mutex_unlock(&vk->queue_mutex);
+	vk_queue_lock(vk->main_queue);
+	ret = vk->vkQueuePresentKHR(vk->main_queue->queue, &present);
+	vk_queue_unlock(vk->main_queue);
 
 	if (ret != VK_SUCCESS) {
 		VK_ERROR(vk, "Error: could not present to queue.\n");

src/xrt/compositor/render/render_resources.c

@@ -595,7 +595,7 @@ render_resources_init(struct render_resources *r,
 	VkCommandPoolCreateInfo command_pool_info = {
 	    .sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
 	    .flags = VK_COMMAND_POOL_CREATE_TRANSIENT_BIT,
-	    .queueFamilyIndex = vk->main_queue.family_index,
+	    .queueFamilyIndex = vk->main_queue->family_index,
 	};
 
 	ret = vk->vkCreateCommandPool(vk->device, &command_pool_info, NULL, &r->cmd_pool);
@@ -657,7 +657,7 @@ render_resources_init(struct render_resources *r,
 		    r->mock.color.image);        // dst
 		VK_CHK_WITH_RET(ret, "prepare_mock_image_locked", false);
 
-		ret = vk_cmd_end_submit_wait_and_free_cmd_buffer_locked(vk, &vk->main_queue, r->cmd_pool, cmd);
+		ret = vk_cmd_end_submit_wait_and_free_cmd_buffer_locked(vk, vk->main_queue, r->cmd_pool, cmd);
 		VK_CHK_WITH_RET(ret, "vk_cmd_end_submit_wait_and_free_cmd_buffer_locked", false);
 
 		// No need to wait, submit waits on the fence.

src/xrt/compositor/util/comp_swapchain.c

@@ -283,9 +283,9 @@ image_cleanup(struct vk_bundle *vk, struct comp_swapchain_image *image)
 	 * validation doesn't complain. This is done during image destruction so
 	 * isn't time critical.
 	 */
-	os_mutex_lock(&vk->queue_mutex);
+	vk_queue_lock(vk->main_queue);
 	vk->vkDeviceWaitIdle(vk->device);
-	os_mutex_unlock(&vk->queue_mutex);
+	vk_queue_unlock(vk->main_queue);
 
 	// The field array_size is shared, only reset once both are freed.
 	image_view_array_cleanup(vk, image->array_size, &image->views.alpha);

tests/tests_comp_client_vulkan.cpp

@@ -152,8 +152,8 @@ TEST_CASE("client_compositor", "[.][needgpu]")
 	    vk->has_KHR_image_format_list, // image_format_list_enabled
 	    false,                         // debug_utils_enabled
 	    false,                         // renderdoc_enabled
-	    vk->main_queue.family_index,   //
-	    vk->main_queue.index);
+	    vk->main_queue->family_index,  //
+	    vk->main_queue->index);
 	struct xrt_compositor *xc = &xcvk->base;
 
 	SECTION("CreateSwapchain calls native create")