Vkqueuewaitidle correct author (#1029)

google/vulkan-performance-layers requires a call to vkDeviceWaitIdle or vkQueueWaitIdle. Since we want to be able to use that layer in conjunction with Amber we need to somehow communicate that the Amber script has completed.

16 files changed, 69 insertions, 17 deletions

diff --git a/include/amber/recipe.h b/include/amber/recipe.h
index 1b83213..0fd2445 100644
--- a/include/amber/recipe.h
+++ b/include/amber/recipe.h
@@ -44,6 +44,9 @@ class RecipeImpl {
   /// Sets the fence timeout value to |timeout_ms|.
   virtual void SetFenceTimeout(uint32_t timeout_ms) = 0;
 
+  /// Sets or clears runtime layer bit to |enabled|.
+  virtual void SetPipelineRuntimeLayerEnabled(bool enabled) = 0;
+
  protected:
   RecipeImpl();
 };
@@ -73,6 +76,9 @@ class Recipe {
   /// Sets the timeout value for fences to |timeout_ms|.
   void SetFenceTimeout(uint32_t timeout_ms);
 
+  /// Sets or clears runtime layer bit to |enabled|.
+  void SetPipelineRuntimeLayerEnabled(bool enabled);
+
  private:
   RecipeImpl* impl_;
 };
diff --git a/samples/amber.cc b/samples/amber.cc
index ec2678c..f6d91c5 100644
--- a/samples/amber.cc
+++ b/samples/amber.cc
@@ -518,6 +518,9 @@ int main(int argc, const char** argv) {
     if (options.fence_timeout > -1)
       recipe->SetFenceTimeout(static_cast<uint32_t>(options.fence_timeout));
 
+    recipe->SetPipelineRuntimeLayerEnabled(
+      options.enable_pipeline_runtime_layer);
+
     recipe_data.emplace_back();
     recipe_data.back().file = file;
     recipe_data.back().recipe = std::move(recipe);
diff --git a/src/engine.h b/src/engine.h
index 7eec6bc..d444258 100644
--- a/src/engine.h
+++ b/src/engine.h
@@ -35,6 +35,7 @@ class VirtualFileStore;
 struct EngineData {
   /// The timeout to use for fences, in milliseconds.
   uint32_t fence_timeout_ms = 10000;
+  bool pipeline_runtime_layer_enabled = false;
 };
 
 /// Abstract class which describes a backing engine for Amber.
diff --git a/src/recipe.cc b/src/recipe.cc
index f7507b9..7e22bd4 100644
--- a/src/recipe.cc
+++ b/src/recipe.cc
@@ -50,4 +50,9 @@ void Recipe::SetFenceTimeout(uint32_t timeout_ms) {
     impl_->SetFenceTimeout(timeout_ms);
 }
 
+void Recipe::SetPipelineRuntimeLayerEnabled(bool enabled) {
+  if (impl_)
+    impl_->SetPipelineRuntimeLayerEnabled(enabled);
+}
+
 }  // namespace amber
diff --git a/src/script.h b/src/script.h
index 196b62e..b4c6e1a 100644
--- a/src/script.h
+++ b/src/script.h
@@ -67,6 +67,11 @@ class Script : public RecipeImpl {
     engine_data_.fence_timeout_ms = timeout_ms;
   }
 
+  /// Sets or clears runtime layer bit to |enabled|.
+  void SetPipelineRuntimeLayerEnabled(bool enabled) override {
+    engine_data_.pipeline_runtime_layer_enabled = enabled;
+  }
+
   /// Adds |pipeline| to the list of known pipelines. The |pipeline| must have
   /// a unique name over all pipelines in the script.
   Result AddPipeline(std::unique_ptr<Pipeline> pipeline) {
diff --git a/src/vulkan/command_buffer.cc b/src/vulkan/command_buffer.cc
index a61c241..b4d28a6 100644
--- a/src/vulkan/command_buffer.cc
+++ b/src/vulkan/command_buffer.cc
@@ -73,7 +73,8 @@ Result CommandBuffer::BeginRecording() {
   return {};
 }
 
-Result CommandBuffer::SubmitAndReset(uint32_t timeout_ms) {
+Result CommandBuffer::SubmitAndReset(uint32_t timeout_ms,
+  bool pipeline_runtime_layer_enabled) {
   if (device_->GetPtrs()->vkEndCommandBuffer(command_) != VK_SUCCESS)
     return Result("Vulkan::Calling vkEndCommandBuffer Fail");
 
@@ -117,6 +118,15 @@ Result CommandBuffer::SubmitAndReset(uint32_t timeout_ms) {
     return Result("Vulkan::Calling vkWaitForFences Fail (" + result_str + ")");
   }
 
+    /*
+  google/vulkan-performance-layers requires a call to vkDeviceWaitIdle or
+  vkQueueWaitIdle in order to report the information. Since we want to be
+  able to use that layer in conjunction with Amber we need to somehow
+  communicate that the Amber script has completed.
+  */
+  if (pipeline_runtime_layer_enabled)
+    device_->GetPtrs()->vkQueueWaitIdle(device_->GetVkQueue());
+
   if (device_->GetPtrs()->vkResetCommandBuffer(command_, 0) != VK_SUCCESS)
     return Result("Vulkan::Calling vkResetCommandBuffer Fail");
 
@@ -141,9 +151,10 @@ CommandBufferGuard::~CommandBufferGuard() {
     buffer_->Reset();
 }
 
-Result CommandBufferGuard::Submit(uint32_t timeout_ms) {
+Result CommandBufferGuard::Submit(uint32_t timeout_ms,
+  bool pipeline_runtime_layer_enabled) {
   assert(buffer_->guarded_);
-  return buffer_->SubmitAndReset(timeout_ms);
+  return buffer_->SubmitAndReset(timeout_ms, pipeline_runtime_layer_enabled);
 }
 
 }  // namespace vulkan
diff --git a/src/vulkan/command_buffer.h b/src/vulkan/command_buffer.h
index 377d052..349cce4 100644
--- a/src/vulkan/command_buffer.h
+++ b/src/vulkan/command_buffer.h
@@ -48,7 +48,8 @@ class CommandBuffer {
   friend CommandBufferGuard;
 
   Result BeginRecording();
-  Result SubmitAndReset(uint32_t timeout_ms);
+  Result SubmitAndReset(uint32_t timeout_ms,
+                        bool pipeline_runtime_layer_enabled);
   void Reset();
 
   bool guarded_ = false;
@@ -83,7 +84,8 @@ class CommandBufferGuard {
   Result GetResult() { return result_; }
 
   /// Submits and resets the internal command buffer.
-  Result Submit(uint32_t timeout_ms);
+  Result Submit(uint32_t timeout_ms,
+                bool pipeline_runtime_layer_enabled);
 
  private:
   Result result_;
diff --git a/src/vulkan/compute_pipeline.cc b/src/vulkan/compute_pipeline.cc
index 61e4916..dd7a990 100644
--- a/src/vulkan/compute_pipeline.cc
+++ b/src/vulkan/compute_pipeline.cc
@@ -23,10 +23,12 @@ namespace vulkan {
 ComputePipeline::ComputePipeline(
     Device* device,
     uint32_t fence_timeout_ms,
+    bool pipeline_runtime_layer_enabled,
     const std::vector<VkPipelineShaderStageCreateInfo>& shader_stage_info)
     : Pipeline(PipelineType::kCompute,
                device,
                fence_timeout_ms,
+               pipeline_runtime_layer_enabled,
                shader_stage_info) {}
 
 ComputePipeline::~ComputePipeline() = default;
@@ -100,7 +102,7 @@ Result ComputePipeline::Compute(uint32_t x, uint32_t y, uint32_t z) {
                                           pipeline);
     device_->GetPtrs()->vkCmdDispatch(command_->GetVkCommandBuffer(), x, y, z);
 
-    r = guard.Submit(GetFenceTimeout());
+    r = guard.Submit(GetFenceTimeout(), GetPipelineRuntimeLayerEnabled());
     if (!r.IsSuccess())
       return r;
   }
diff --git a/src/vulkan/compute_pipeline.h b/src/vulkan/compute_pipeline.h
index 750b004..d6597be 100644
--- a/src/vulkan/compute_pipeline.h
+++ b/src/vulkan/compute_pipeline.h
@@ -30,6 +30,7 @@ class ComputePipeline : public Pipeline {
   ComputePipeline(
       Device* device,
       uint32_t fence_timeout_ms,
+      bool pipeline_runtime_layer_enabled,
       const std::vector<VkPipelineShaderStageCreateInfo>& shader_stage_info);
   ~ComputePipeline() override;
 
diff --git a/src/vulkan/engine_vulkan.cc b/src/vulkan/engine_vulkan.cc
index 1ed60f4..b0842ec 100644
--- a/src/vulkan/engine_vulkan.cc
+++ b/src/vulkan/engine_vulkan.cc
@@ -170,7 +170,8 @@ Result EngineVulkan::CreatePipeline(amber::Pipeline* pipeline) {
   std::unique_ptr<Pipeline> vk_pipeline;
   if (pipeline->GetType() == PipelineType::kCompute) {
     vk_pipeline = MakeUnique<ComputePipeline>(
-        device_.get(), engine_data.fence_timeout_ms, stage_create_info);
+        device_.get(), engine_data.fence_timeout_ms,
+        engine_data.pipeline_runtime_layer_enabled, stage_create_info);
     r = vk_pipeline->AsCompute()->Initialize(pool_.get());
     if (!r.IsSuccess())
       return r;
@@ -178,7 +179,8 @@ Result EngineVulkan::CreatePipeline(amber::Pipeline* pipeline) {
     vk_pipeline = MakeUnique<GraphicsPipeline>(
         device_.get(), pipeline->GetColorAttachments(),
         pipeline->GetDepthStencilBuffer(), pipeline->GetResolveTargets(),
-        engine_data.fence_timeout_ms, stage_create_info);
+        engine_data.fence_timeout_ms,
+        engine_data.pipeline_runtime_layer_enabled, stage_create_info);
 
     vk_pipeline->AsGraphics()->SetPatchControlPoints(
         pipeline->GetPipelineData()->GetPatchControlPoints());
diff --git a/src/vulkan/graphics_pipeline.cc b/src/vulkan/graphics_pipeline.cc
index 138f32c..485ebb6 100644
--- a/src/vulkan/graphics_pipeline.cc
+++ b/src/vulkan/graphics_pipeline.cc
@@ -392,10 +392,12 @@ GraphicsPipeline::GraphicsPipeline(
     amber::Pipeline::BufferInfo depth_stencil_buffer,
     const std::vector<amber::Pipeline::BufferInfo>& resolve_targets,
     uint32_t fence_timeout_ms,
+    bool pipeline_runtime_layer_enabled,
     const std::vector<VkPipelineShaderStageCreateInfo>& shader_stage_info)
     : Pipeline(PipelineType::kGraphics,
                device,
                fence_timeout_ms,
+               pipeline_runtime_layer_enabled,
                shader_stage_info),
       depth_stencil_buffer_(depth_stencil_buffer) {
   for (const auto& info : color_buffers)
@@ -778,7 +780,7 @@ Result GraphicsPipeline::SetIndexBuffer(Buffer* buffer) {
   if (!r.IsSuccess())
     return r;
 
-  return guard.Submit(GetFenceTimeout());
+  return guard.Submit(GetFenceTimeout(), GetPipelineRuntimeLayerEnabled());
 }
 
 Result GraphicsPipeline::SetClearColor(float r, float g, float b, float a) {
@@ -869,7 +871,8 @@ Result GraphicsPipeline::Clear() {
 
   frame_->TransferImagesToHost(command_.get());
 
-  Result r = cmd_buf_guard.Submit(GetFenceTimeout());
+  Result r = cmd_buf_guard.Submit(GetFenceTimeout(),
+                                  GetPipelineRuntimeLayerEnabled());
   if (!r.IsSuccess())
     return r;
 
@@ -958,7 +961,8 @@ Result GraphicsPipeline::Draw(const DrawArraysCommand* command,
 
     frame_->TransferImagesToHost(command_.get());
 
-    r = cmd_buf_guard.Submit(GetFenceTimeout());
+    r = cmd_buf_guard.Submit(GetFenceTimeout(),
+                             GetPipelineRuntimeLayerEnabled());
     if (!r.IsSuccess())
       return r;
   }
diff --git a/src/vulkan/graphics_pipeline.h b/src/vulkan/graphics_pipeline.h
index cd55aad..4bc5f7d 100644
--- a/src/vulkan/graphics_pipeline.h
+++ b/src/vulkan/graphics_pipeline.h
@@ -45,6 +45,7 @@ class GraphicsPipeline : public Pipeline {
       amber::Pipeline::BufferInfo depth_stencil_buffer,
       const std::vector<amber::Pipeline::BufferInfo>& resolve_targets,
       uint32_t fence_timeout_ms,
+      bool pipeline_runtime_layer_enabled,
       const std::vector<VkPipelineShaderStageCreateInfo>&);
   ~GraphicsPipeline() override;
 
diff --git a/src/vulkan/pipeline.cc b/src/vulkan/pipeline.cc
index 8cd4995..d03b2de 100644
--- a/src/vulkan/pipeline.cc
+++ b/src/vulkan/pipeline.cc
@@ -40,11 +40,13 @@ Pipeline::Pipeline(
     PipelineType type,
     Device* device,
     uint32_t fence_timeout_ms,
+    bool    pipeline_runtime_layer_enabled,
     const std::vector<VkPipelineShaderStageCreateInfo>& shader_stage_info)
     : device_(device),
       pipeline_type_(type),
       shader_stage_info_(shader_stage_info),
-      fence_timeout_ms_(fence_timeout_ms) {}
+      fence_timeout_ms_(fence_timeout_ms),
+      pipeline_runtime_layer_enabled_(pipeline_runtime_layer_enabled) {}
 
 Pipeline::~Pipeline() {
   // Command must be reset before we destroy descriptors or we get a validation
@@ -441,7 +443,8 @@ Result Pipeline::SendDescriptorDataToDeviceIfNeeded() {
     // done after resizing backed buffer i.e., copying data to the new
     // buffer from the old one. Thus, we must submit commands here to
     // guarantee this.
-    Result r = guard.Submit(GetFenceTimeout());
+    Result r = guard.Submit(GetFenceTimeout(),
+                            GetPipelineRuntimeLayerEnabled());
     if (!r.IsSuccess())
       return r;
   }
@@ -473,7 +476,7 @@ Result Pipeline::SendDescriptorDataToDeviceIfNeeded() {
           "this should be unreachable");
     }
   }
-  return guard.Submit(GetFenceTimeout());
+  return guard.Submit(GetFenceTimeout(), GetPipelineRuntimeLayerEnabled());
 }
 
 void Pipeline::BindVkDescriptorSets(const VkPipelineLayout& pipeline_layout) {
@@ -548,7 +551,8 @@ Result Pipeline::ReadbackDescriptorsToHostDataQueue() {
       }
     }
 
-    Result r = guard.Submit(GetFenceTimeout());
+    Result r = guard.Submit(GetFenceTimeout(),
+                            GetPipelineRuntimeLayerEnabled());
     if (!r.IsSuccess())
       return r;
   }
diff --git a/src/vulkan/pipeline.h b/src/vulkan/pipeline.h
index 1f85cb6..58cefcb 100644
--- a/src/vulkan/pipeline.h
+++ b/src/vulkan/pipeline.h
@@ -78,6 +78,7 @@ class Pipeline {
       PipelineType type,
       Device* device,
       uint32_t fence_timeout_ms,
+      bool    pipeline_runtime_layer_enabled,
       const std::vector<VkPipelineShaderStageCreateInfo>& shader_stage_info);
 
   /// Initializes the pipeline.
@@ -101,6 +102,8 @@ class Pipeline {
 
   const char* GetEntryPointName(VkShaderStageFlagBits stage) const;
   uint32_t GetFenceTimeout() const { return fence_timeout_ms_; }
+  bool     GetPipelineRuntimeLayerEnabled()
+       const { return pipeline_runtime_layer_enabled_; }
 
   Result CreateVkPipelineLayout(VkPipelineLayout* pipeline_layout);
 
@@ -134,6 +137,7 @@ class Pipeline {
   std::vector<Buffer*> descriptor_buffers_;
 
   uint32_t fence_timeout_ms_ = 1000;
+  bool pipeline_runtime_layer_enabled_ = false;
   bool descriptor_related_objects_already_created_ = false;
   std::unordered_map<VkShaderStageFlagBits,
                      std::string,
diff --git a/src/vulkan/pipeline_test.cc b/src/vulkan/pipeline_test.cc
index 9f60174..a6982fd 100644
--- a/src/vulkan/pipeline_test.cc
+++ b/src/vulkan/pipeline_test.cc
@@ -26,7 +26,7 @@ using VulkanPipelineTest = testing::Test;
 TEST_F(VulkanPipelineTest, AddBufferDescriptorAddPushConstant) {
   amber::Pipeline amber_pipeline(PipelineType::kCompute);
   std::vector<VkPipelineShaderStageCreateInfo> create_infos;
-  ComputePipeline pipeline(nullptr, 0, create_infos);
+  ComputePipeline pipeline(nullptr, 0, false, create_infos);
 
   auto cmd = MakeUnique<BufferCommand>(BufferCommand::BufferType::kPushConstant,
                                        &amber_pipeline);
@@ -38,7 +38,7 @@ TEST_F(VulkanPipelineTest, AddBufferDescriptorAddPushConstant) {
 TEST_F(VulkanPipelineTest, AddBufferDescriptorAddBufferTwice) {
   amber::Pipeline amber_pipeline(PipelineType::kCompute);
   std::vector<VkPipelineShaderStageCreateInfo> create_infos;
-  ComputePipeline pipeline(nullptr, 0, create_infos);
+  ComputePipeline pipeline(nullptr, 0, false, create_infos);
 
   auto cmd = MakeUnique<BufferCommand>(BufferCommand::BufferType::kUniform,
                                        &amber_pipeline);
diff --git a/src/vulkan/vk-funcs-1-0.inc b/src/vulkan/vk-funcs-1-0.inc
index 22c37e8..b5a7ac0 100644
--- a/src/vulkan/vk-funcs-1-0.inc
+++ b/src/vulkan/vk-funcs-1-0.inc
@@ -63,3 +63,4 @@ AMBER_VK_FUNC(vkResetFences)
 AMBER_VK_FUNC(vkUnmapMemory)
 AMBER_VK_FUNC(vkUpdateDescriptorSets)
 AMBER_VK_FUNC(vkWaitForFences)
+AMBER_VK_FUNC(vkQueueWaitIdle)