a/vk: refactor vk_cmd(_pool) utils to support different queue (families)

Part-of: <https://gitlab.freedesktop.org/monado/monado/-/merge_requests/2572>

src/xrt/auxiliary/vk/vk_cmd.c

@@ -78,12 +78,13 @@ err_buffer:
 }
 
 XRT_CHECK_RESULT VkResult
-vk_cmd_submit_locked(struct vk_bundle *vk, uint32_t count, const VkSubmitInfo *infos, VkFence fence)
+vk_cmd_submit_locked(
+    struct vk_bundle *vk, struct vk_bundle_queue *queue, uint32_t count, const VkSubmitInfo *infos, VkFence fence)
 {
 	VkResult ret;
 
 	os_mutex_lock(&vk->queue_mutex);
-	ret = vk->vkQueueSubmit(vk->main_queue.queue, count, infos, fence);
+	ret = vk->vkQueueSubmit(queue->queue, count, infos, fence);
 	os_mutex_unlock(&vk->queue_mutex);
 
 	if (ret != VK_SUCCESS) {
@@ -94,7 +95,10 @@ vk_cmd_submit_locked(struct vk_bundle *vk, uint32_t count, const VkSubmitInfo *i
 }
 
 XRT_CHECK_RESULT VkResult
-vk_cmd_end_submit_wait_and_free_cmd_buffer_locked(struct vk_bundle *vk, VkCommandPool pool, VkCommandBuffer cmd_buffer)
+vk_cmd_end_submit_wait_and_free_cmd_buffer_locked(struct vk_bundle *vk,
+                                                  struct vk_bundle_queue *queue,
+                                                  VkCommandPool pool,
+                                                  VkCommandBuffer cmd_buffer)
 {
 	VkFence fence;
 	VkResult ret;
@@ -129,7 +133,7 @@ vk_cmd_end_submit_wait_and_free_cmd_buffer_locked(struct vk_bundle *vk, VkComman
 	    .pCommandBuffers = &cmd_buffer,
 	};
 
-	ret = vk_cmd_submit_locked(vk, 1, &submitInfo, fence);
+	ret = vk_cmd_submit_locked(vk, queue, 1, &submitInfo, fence);
 	if (ret != VK_SUCCESS) {
 		VK_ERROR(vk, "vk_cmd_pool_submit_locked: %s", vk_result_string(ret));
 		goto out_fence;

src/xrt/auxiliary/vk/vk_cmd.h

@@ -167,7 +167,8 @@ vk_cmd_create_and_begin_cmd_buffer_locked(struct vk_bundle *vk,
  * @ingroup aux_vk
  */
 XRT_CHECK_RESULT VkResult
-vk_cmd_submit_locked(struct vk_bundle *vk, uint32_t count, const VkSubmitInfo *infos, VkFence fence);
+vk_cmd_submit_locked(
+    struct vk_bundle *vk, struct vk_bundle_queue *queue, uint32_t count, const VkSubmitInfo *infos, VkFence fence);
 
 /*!
  * A do everything command buffer submission function, the `_locked` suffix
@@ -189,7 +190,10 @@ vk_cmd_submit_locked(struct vk_bundle *vk, uint32_t count, const VkSubmitInfo *i
  * @ingroup aux_vk
  */
 XRT_CHECK_RESULT VkResult
-vk_cmd_end_submit_wait_and_free_cmd_buffer_locked(struct vk_bundle *vk, VkCommandPool pool, VkCommandBuffer cmd_buffer);
+vk_cmd_end_submit_wait_and_free_cmd_buffer_locked(struct vk_bundle *vk,
+                                                  struct vk_bundle_queue *queue,
+                                                  VkCommandPool pool,
+                                                  VkCommandBuffer cmd_buffer);
 
 
 /*

src/xrt/auxiliary/vk/vk_cmd_pool.c

@@ -19,9 +19,13 @@
  */
 
 XRT_CHECK_RESULT VkResult
-vk_cmd_pool_init(struct vk_bundle *vk, struct vk_cmd_pool *pool, VkCommandPoolCreateFlags flags)
+vk_cmd_pool_init_for_queue(struct vk_bundle *vk,
+                           struct vk_cmd_pool *pool,
+                           VkCommandPoolCreateFlags flags,
+                           struct vk_bundle_queue *queue)
 {
 	VkResult ret;
+	assert(queue);
 
 	XRT_MAYBE_UNUSED int iret = os_mutex_init(&pool->mutex);
 	assert(iret == 0);
@@ -29,7 +33,7 @@ vk_cmd_pool_init(struct vk_bundle *vk, struct vk_cmd_pool *pool, VkCommandPoolCr
 	VkCommandPoolCreateInfo cmd_pool_info = {
 	    .sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
 	    .flags = flags,
-	    .queueFamilyIndex = vk->main_queue.family_index,
+	    .queueFamilyIndex = queue->family_index,
 	};
 
 	ret = vk->vkCreateCommandPool(vk->device, &cmd_pool_info, NULL, &pool->pool);
@@ -38,6 +42,8 @@ vk_cmd_pool_init(struct vk_bundle *vk, struct vk_cmd_pool *pool, VkCommandPoolCr
 		os_mutex_destroy(&pool->mutex);
 	}
 
+	pool->queue = queue;
+
 	return ret;
 }
 
@@ -133,7 +139,7 @@ vk_cmd_pool_submit_cmd_buffer_locked(struct vk_bundle *vk, struct vk_cmd_pool *p
 	    .pCommandBuffers = &cmd_buffer,
 	};
 
-	ret = vk_cmd_submit_locked(vk, 1, &submitInfo, VK_NULL_HANDLE);
+	ret = vk_cmd_submit_locked(vk, pool->queue, 1, &submitInfo, VK_NULL_HANDLE);
 
 	if (ret != VK_SUCCESS) {
 		VK_ERROR(vk, "vk_cmd_submit_locked: %s", vk_result_string(ret));

src/xrt/auxiliary/vk/vk_cmd_pool.h

@@ -31,7 +31,18 @@ extern "C" {
  */
 struct vk_cmd_pool
 {
+	//! The command pool for command buffers
 	VkCommandPool pool;
+
+	/*!
+	 * @brief Queue (family) associated with @ref vk_cmd_pool::pool,
+	 *
+	 * weak reference to any queue in @ref vk_bundle (e.g. vk_bundle::[graphics|compute]_queue)
+	 * should not live longer than the @ref vk_bundle instance.
+	 */
+	struct vk_bundle_queue *queue;
+
+	//! Command Pool mutex
 	struct os_mutex mutex;
 };
 
@@ -48,7 +59,21 @@ struct vk_cmd_pool
  * @public @memberof vk_cmd_pool
  */
 XRT_CHECK_RESULT VkResult
-vk_cmd_pool_init(struct vk_bundle *vk, struct vk_cmd_pool *pool, VkCommandPoolCreateFlags flags);
+vk_cmd_pool_init_for_queue(struct vk_bundle *vk,
+                           struct vk_cmd_pool *pool,
+                           VkCommandPoolCreateFlags flags,
+                           struct vk_bundle_queue *queue);
+
+/*!
+ * Create a command buffer pool.
+ *
+ * @public @memberof vk_cmd_pool
+ */
+static inline XRT_CHECK_RESULT VkResult
+vk_cmd_pool_init(struct vk_bundle *vk, struct vk_cmd_pool *pool, VkCommandPoolCreateFlags flags)
+{
+	return vk_cmd_pool_init_for_queue(vk, pool, flags, &vk->main_queue);
+}
 
 /*!
  * Destroy a command buffer pool, lock must not be held, externally
@@ -113,7 +138,7 @@ vk_cmd_pool_end_submit_wait_and_free_cmd_buffer_locked(struct vk_bundle *vk,
                                                        struct vk_cmd_pool *pool,
                                                        VkCommandBuffer cmd_buffer)
 {
-	return vk_cmd_end_submit_wait_and_free_cmd_buffer_locked(vk, pool->pool, cmd_buffer);
+	return vk_cmd_end_submit_wait_and_free_cmd_buffer_locked(vk, pool->queue, pool->pool, cmd_buffer);
 }
 
 /*!
@@ -184,7 +209,7 @@ vk_cmd_pool_submit(
     struct vk_bundle *vk, struct vk_cmd_pool *pool, uint32_t count, const VkSubmitInfo *infos, VkFence fence)
 {
 	vk_cmd_pool_lock(pool);
-	VkResult ret = vk_cmd_submit_locked(vk, count, infos, fence);
+	VkResult ret = vk_cmd_submit_locked(vk, pool->queue, count, infos, fence);
 	vk_cmd_pool_unlock(pool);
 	return ret;
 }

src/xrt/compositor/main/comp_renderer.c

@@ -665,7 +665,7 @@ renderer_submit_queue(struct comp_renderer *r, VkCommandBuffer cmd, VkPipelineSt
 	 * us avoid taking a lot of locks. The queue lock will be taken by
 	 * @ref vk_cmd_submit_locked tho.
 	 */
-	ret = vk_cmd_submit_locked(vk, 1, &comp_submit_info, r->fences[r->acquired_buffer]);
+	ret = vk_cmd_submit_locked(vk, &vk->main_queue, 1, &comp_submit_info, r->fences[r->acquired_buffer]);
 
 	// We have now completed the submit, even if we failed.
 	comp_target_mark_submit_end(ct, frame_id, os_monotonic_get_ns());

src/xrt/compositor/main/comp_window_peek.c

@@ -434,7 +434,7 @@ comp_window_peek_blit(struct comp_window_peek *w, VkImage src, int32_t width, in
 	};
 
 	// Done writing commands, submit to queue.
-	ret = vk_cmd_submit_locked(vk, 1, &submit, VK_NULL_HANDLE);
+	ret = vk_cmd_submit_locked(vk, &vk->main_queue, 1, &submit, VK_NULL_HANDLE);
 
 	// Done submitting commands, unlock pool.
 	vk_cmd_pool_unlock(&w->pool);

src/xrt/compositor/render/render_resources.c

@@ -657,7 +657,7 @@ render_resources_init(struct render_resources *r,
 		    r->mock.color.image);        // dst
 		VK_CHK_WITH_RET(ret, "prepare_mock_image_locked", false);
 
-		ret = vk_cmd_end_submit_wait_and_free_cmd_buffer_locked(vk, r->cmd_pool, cmd);
+		ret = vk_cmd_end_submit_wait_and_free_cmd_buffer_locked(vk, &vk->main_queue, r->cmd_pool, cmd);
 		VK_CHK_WITH_RET(ret, "vk_cmd_end_submit_wait_and_free_cmd_buffer_locked", false);
 
 		// No need to wait, submit waits on the fence.