VulkanWrapper.cpp

/* ----------------------
* Copyright 2023 Université Libre de Bruxelles(ULB), Universidad Politécnica de Madrid(UPM), CREAL, Deutsches Zentrum für Luft - und Raumfahrt(DLR)
 
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at < http://www.apache.org/licenses/LICENSE-2.0>
 
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissionsand
* limitations under the License.
---------------------- */
 
 
#include "VulkanWrapper.h"
#include "../WindowAbstract.h"
//#include "../NoWindow.h"
#include "Shader.h"
#include <iostream>
#include <fstream>
#include "VulkanHelperFunction.h"
#include "InputProvider.h"
 
#include <nlohmann/json.hpp>
 
using json = nlohmann::json;
 
#ifdef __ANDROID__
static const char* kTAG = "renderPass";
#define LOGI(...) \
  ((void)__android_log_print(ANDROID_LOG_INFO, kTAG, __VA_ARGS__))
#endif
 
#ifdef __ANDROID__
VulkanWrapper::VulkanWrapper(WindowAbstract* window,  android_app* appAndroid) :context(window), renderPass(&context, this), drawing(&context,&renderPass, window, this), multiviewSetup(window->useOpenXR())
{
    this->window = window;
    window->setWrapper(this);
    this->appAndroid = appAndroid;
}
#else
VulkanWrapper::VulkanWrapper(WindowAbstract* window, PipelineMode pmode) :
  context(window),
  renderPass([&]()->VulkanRenderPassAbstract*{
             switch (pmode) {
              default:
               return new VulkanRenderPass(&context, this);
             }
             }()),
  drawing(&context, renderPass.get(), window, this),
  multiviewSetup(window->useOpenXR())
{
    this->window = window;
    window->setWrapper(this);
    
}
#endif
 
#ifdef HVT_UDP_CONTROL
void VulkanWrapper::init(unsigned short port)
{
#else
void VulkanWrapper::init()
{
#endif
 
    window->initWindow();
#ifdef __ANDROID__
    ShadersList::getInstance().loadShaders(appAndroid);
#else
    ShadersList::getInstance().loadShaders();
#endif
    if (!context.isInitialized()) {
        D(PRINT("====context======="));
        context.initVulkan();
    }
    D(PRINT("===continue window initialisation==="));
    window->continueInit();
    //set the number of attachment to 
    attachmentSize = window->getAttachmentSize();
    int viewNum = window->getViewNumber();
 
#ifdef HVT_UDP_CONTROL
    params.init(viewNum, window, this, port);
    this->port = port;
#else
    params.init(viewNum, window, this);
#endif // DEBUG
 
    D(PRINT("Size of attachment = %d", attachmentSize));
}
 
std::vector<vk::Format> VulkanWrapper::getRenderPassSupportedDepthFormats() const {
  return renderPass->getSupportedDepthFormats();
}
 
void VulkanWrapper::endInit(InputProvider * inputProvider, StartingPosition startingPt){
    this->inputProvider = inputProvider;
    auto input = inputProvider->enumerateStreamsParameters();
    assert(input.size() >= 1);
 
    correctionVectorRotation.resize(input.size());
    correctionVectorTranslation.resize(input.size());
    correctionPrincipalPt.resize(input.size());
    correctionFocal.resize(input.size());
 
 
    streamFrameInfos.resize(input.size());
    cameraActivation.resize(input.size(),true);
 
    float maxD = input[0].farDepth;
    float minD = input[0].nearDepth;
    for (const auto & in : input) {
        maxD = in.farDepth > maxD ? in.farDepth : maxD;
        minD = in.nearDepth > minD ? in.nearDepth : minD;
    }
    params.max_depth = maxD;
    params.min_depth = minD;
    
    
    D(PRINT("=====renderpass======"));
    renderPass->init(inputProvider);
    D(PRINT("=====drawing======"));
    drawing.init(inputProvider);
    
    if (startingPt == StartingPosition::Average) {
        requireAverageInputPosition = true;
    }
    else {
        requireAverageInputPosition = false;
    }
}
 
 
void VulkanWrapper::saveConfigurationsToDisk()
{
    
    std::cout << "Save correction of rotation to disk at " << fileN << std::endl;
    std::ofstream stream(fileN, std::ifstream::out);
 
    assert(correctionVectorRotation.size() == correctionVectorTranslation.size());
    assert(correctionVectorTranslation.size() == correctionPrincipalPt.size());
    assert(correctionPrincipalPt.size() == correctionFocal.size());
 
    
    json j;
    for (int id = 0; id < correctionVectorRotation.size(); id ++ ) {
        //j["camera"][id] = json::array();
        j["camera"][id]["id"] = id;
        
        //j["camera"][id]["correction"] = json::object();
        auto r = correctionVectorRotation[id];
        j["camera"][id]["correction"]["rotation"] = { r.x,r.y,r.z };
        auto t = correctionVectorTranslation[id];
        j["camera"][id]["correction"]["translation"] = {t.x,t.y,t.z};
        auto p = correctionPrincipalPt[id];
        j["camera"][id]["correction"]["principalPt"] = json::array({p.x,p.y});
        auto f = correctionFocal[id];
        j["camera"][id]["correction"]["focal"] = json::array({f.x,f.y});
 
        //j["camera"][id]["current"] = json::array();
        auto ro = streamFrameInfos[id].extrinsics.rotation;
        j["camera"][id]["current"]["rotation"] = {ro.x,ro.y,ro.z};
        auto to = streamFrameInfos[id].extrinsics.position;
        j["camera"][id]["current"]["translation"] = { to.x,to.y,to.z };
 
        //-------- TODO case with equirectangular 
        if (streamFrameInfos[id].intrinsics.index() == 0) {
            auto po = std::get<InputProvider::PerspectiveIntrinsics>(streamFrameInfos[id].intrinsics).principle;
            j["camera"][id]["current"]["principalPt"] = json::array({ po.x,po.y });
            auto fo = std::get<InputProvider::PerspectiveIntrinsics>(streamFrameInfos[id].intrinsics).focals;
            j["camera"][id]["current"]["focal"] = json::array({ fo.x,fo.y });
        }
 
    }
    stream << j.dump(4);
 
    stream.close();
}
 
void VulkanWrapper::loadConfigurationsFromDisk()
{
    
    std::ifstream stream(fileN, std::ifstream::out);
 
    json j;
    std::cout << "load correction of rotation to disk at " << fileN << std::endl;
    if (stream.is_open()) {
        j << stream;
 
        for (auto& cam : j["camera"]) {
            int id = cam["id"];
            auto r = j["camera"][id]["correction"]["rotation"];
            correctionVectorRotation[id] = { r[0],r[1],r[2] };
            auto t = j["camera"][id]["correction"]["translation"];
            correctionVectorTranslation[id] = { t[0],t[1],t[2] };
            auto p = j["camera"][id]["correction"]["principalPt"];
            correctionPrincipalPt[id] = { p[0],p[1] };
            auto f = j["camera"][id]["correction"]["focal"];
            correctionFocal[id] = { f[0],f[1] };
        }
    }
    else {
        std::cout << "No save file found" << std::endl;
    }
}
 
std::vector<bool> VulkanWrapper::getCameraActivation()
{
    return cameraActivation;
}
 
void VulkanWrapper::toggleCamera(int i)
{
    
    assert( i < cameraActivation.size() );
 
    if (!cameraActivation[i] || renderPass->inputImage > 1 ) {
#ifdef HVT_UDP_CONTROL
 
        mutexAct.lock();
#endif // HVT_UDP_CONTROL
 
        requestChange = true;
        cameraActivation[i] = !cameraActivation[i];
#ifdef HVT_UDP_CONTROL
        mutexAct.unlock();
#endif
    }
    else {
        //we don't need to desactivate all the camera
        //always let one activated
        std::cout << "Impossible to desactivate camera " << i << " :there must always be one camera active" << std::endl;
    }
}
 
void VulkanWrapper::setCameraActivation(std::vector<bool> activation)
{
    
    assert(activation.size() == cameraActivation.size());
 
    bool valid = false;
    for (auto a : activation) {
        valid += a;
    }
 
    if (valid) {
#ifdef HVT_UDP_CONTROL
        mutexAct.lock();
#endif
        requestChange = true;
        cameraActivation = activation;
#ifdef HVT_UDP_CONTROL
        mutexAct.unlock();
#endif
    }
    else {
        PRINT("Invalid activation vector. At least one camera should be activated");
    }
}
 
void VulkanWrapper::start()
{
 
    window->mainLoop(&drawing);
    context.device.waitIdle();
}
 
void VulkanWrapper::cleanUp()
{
 
    if(drawing.isInitialized()) {
        drawing.cleanup();
    }
 
    //pipeline.cleanUp();
    if(renderPass->isInitialized()){
        renderPass->cleanUp();
    }
    /*
    if (!window->isSwapChainNeeded()) {
        static_cast<NoWindow*>(window)->cleanUpRessources();
    }*/
    window->cleanUp();
 
}
 
void VulkanWrapper::recreateSwapChain()
{
#ifdef __ANDROID__
    LOGI("recreate swapchain");
#else
    std::cout << ("recreation swapchain !") << std::endl;
#endif
    window->checkForCorrectSize();
    //could be better TODO -> https://vulkan-tutorial.com/Drawing_a_triangle/Swap_chain_recreation
    context.device.waitIdle();
    //cleanUp
    cleanUp();
    //init
#ifdef HVT_UDP_CONTROL
    init(port);
#else
    init();
#endif
 
}
 
void VulkanWrapper::updateSpaceTransform(glm::vec3 translation, glm::vec3 rotation)
{
 
    params.updateSpaceTransform(translation,rotation);
 
}
 
void VulkanWrapper::waitIdle() {
    context.device.waitIdle();
}
void VulkanWrapper::createImageCall(uint32_t width, uint32_t height, vk::Format format, vk::ImageTiling tiling, vk::ImageUsageFlags usage, vk::MemoryPropertyFlags properties, vk::Image& image, vk::DeviceMemory& imageMemory)
{
    createImage(context.device,context.physicalDevice,width,height,format,tiling,usage,properties,image,imageMemory);
}
void VulkanWrapper::createBufferCall(vk::DeviceSize size, vk::BufferUsageFlags usage, vk::MemoryPropertyFlags properties, vk::Buffer& buffer, vk::DeviceMemory& bufferMemory)
{
    createBuffer(size, usage, properties, buffer, bufferMemory, context.device, context.physicalDevice);
}
void VulkanWrapper::copyMemory(VkDeviceSize & size, vk::DeviceMemory& memory, unsigned char* dataToCopy)
{
    void* data;
    context.device.mapMemory(memory, 0, size, {}, &data);
    memcpy(dataToCopy, data, static_cast<size_t>(size));
    context.device.unmapMemory(memory);
}
 
void VulkanWrapper::destroyImageAndMemory(std::vector<vk::Image>& vectorImage, std::vector<vk::DeviceMemory>& vectorMemory)
{
    //delete image
    for (auto const& image : vectorImage) {
        context.device.destroyImage(image);
    }
    //delete attachement memory
    for (auto const& mem : vectorMemory) {
        context.device.freeMemory(mem);
    }
}
 
void VulkanWrapper::destroyBufferAndMemory(std::vector<vk::Buffer>& vectorBuffer, std::vector<vk::DeviceMemory>& vectorMemory)
{
    //delete image
    for (auto const& buf : vectorBuffer) {
        context.device.destroyBuffer(buf);
    }
    //delete attachement memory
    for (auto const& mem : vectorMemory) {
        context.device.freeMemory(mem);
    }
}
 
bool VulkanWrapper::isDepthOutputRecquired()
{
    return window->isDepthRecquired();
}
 
void VulkanWrapper::setBlendingFactor(float blendingfactor)
{
    params.blendingFactor = blendingfactor;
}
 
void VulkanWrapper::setTriangleThreshold(float triangleThreshold)
{
    assert(triangleThreshold > 0);
    params.pixelThreshold = triangleThreshold;
}
 
bool VulkanWrapper::isUsingDifferentViewSize()
{
    assert(params.virtualViewDef.size()  > 0);
    if (params.virtualViewDef.size() == 1) {
        return false;
    }
    else {
        auto ref = params.virtualViewDef[0];
        for (auto & v : params.virtualViewDef) {
            if (v[0] != ref[0] || v[1] != ref[1]) {
                return true;
            }
        }
        return false;
    }
    
}
 
VulkanContext* VulkanWrapper::getContext()
{
    return &context;
}
 
vk::Extent2D VulkanWrapper::getSwapchainExtend(int view)
{
    return window->getSwapchainExtent(view);
}
 
vk::Extent2D VulkanWrapper::getRenderOutputExtent()
{
    /*if (window->isIndepFromWindowDimension()) {
        //return VirtualView [0]
        auto size = app->getConfig().params_virtual[0].getSize();
        return vk::Extent2D(size.width,size.height);
    }
    else { */
        //return the size of the first swapchain
        //return window->swapChainExtent[0];
    //}
        vk::Extent2D max = {0,0};
        for (int i = 0; i < window->getViewNumber(); i++) {
            auto ext = window->getSwapchainExtent(i);
            if (max.width < ext.width) {
                max.width = ext.width;
            }
            if (max.height < ext.height) {
                max.height = ext.height;
            }
        }
        return max;
}
 
vk::Format VulkanWrapper::getSwapchainFormat(int view)
{
    return window->getSwapchainFormat(view);//window->swapChainImageFormat[view];
}
 
vk::Format VulkanWrapper::getDepthSwapchainFormat(int view)
{
    return window->getDepthSwapchainFormat(view);
}
 
vk::ImageLayout VulkanWrapper::getLayoutAfterRenderpass()
{
    auto res = vk::ImageLayout::eColorAttachmentOptimal;
    if (!window->useOpenXR() && !window->isIndepFromWindowDimension()) {
        res = vk::ImageLayout::ePresentSrcKHR;
    }
    return res;
}
 
int VulkanWrapper::getAttachmentSize()
{
    return window->getAttachmentSize();
}
 
int VulkanWrapper::getViewNumber()
{
    return window->getViewNumber();
}
 
void VulkanWrapper::setAverageInputPosition(std::vector<InputProvider::StreamFrameInfo>& frameInfo)
{
    glm::vec3 averagePos = {0,0,0};
    glm::vec3 averageRot = {0,0,0};
    const int numInput = frameInfo.size();
 
    for (auto f : frameInfo) {
        averagePos += f.extrinsics.position;
        //averageRot += f.extrinsics.rotation;
    }
    averagePos /= numInput;
    //averageRot /= numInput;
 
    //convert from OMAF to the coordinate system used by virtual parameters
    auto conversion = glm::transpose(params.conversionToOMAF);
    auto pos  =  conversion * averagePos;
 
    params.setStartingPt(pos, averageRot);
    //params.startingRotation = conversion * averageRot * glm::transpose(conversion) ;
 
    requireAverageInputPosition = false;
}
 
void VulkanWrapper::addDeltaToStartingPoint(glm::vec3 dPos, glm::vec3 dRot)
{
    auto R = glmEulerAnglesDegreeToRotationMatrixNotOMAF(params.getStartingRotation());
    auto dR = glmEulerAnglesDegreeToRotationMatrixNotOMAF(dRot);
 
    auto pos = params.getStartingPosition() + glm::transpose(R)* dPos;
    auto rot = glm::transpose(R) * dRot * R;
 
    params.setStartingPt(pos,rot);
 
 
}
 
void VulkanWrapper::setCorrectionRotation(int camId, glm::vec3 rotationCorrection)
{
    if (camId < correctionVectorRotation.size()) {
        correctionVectorRotation[camId] += rotationCorrection;
    }
    else {
        PRINT("Camera %i does not exist, only %i camera are availables", camId, (int) correctionVectorRotation.size());
    }
}
 
void VulkanWrapper::setCorrectionTranslation(int camId, glm::vec3 translationCorrection)
{
    if (camId < correctionVectorTranslation.size()) {
        correctionVectorTranslation[camId] += translationCorrection;
    }
    else {
        PRINT("Camera %i does not exist, only %i camera are availables", camId, (int)correctionVectorTranslation.size());
    }
}
 
void VulkanWrapper::setCorrectionFocal(int camId, glm::vec2 focals)
{
    if (camId < correctionFocal.size()) {
        correctionFocal[camId] += focals;
    }
    else {
        PRINT("Camera %i does not exist, only %i camera are availables", camId, (int)correctionFocal.size());
    }
}
 
void VulkanWrapper::setCorrectionPrincipalPt(int camId, glm::vec2 principalPt)
{
    if (camId < correctionFocal.size()) {
        correctionPrincipalPt[camId] += principalPt;
    }
    else {
        PRINT("Camera %i does not exist, only %i camera are availables", camId, (int)correctionPrincipalPt.size());
    }
}
 
void VulkanWrapper::setScaleFactor(float scaleFactor)
{
    assert(scaleFactor > 0);
    params.scaleFactor = scaleFactor;
}
 
const std::vector<glm::vec3> VulkanWrapper::getCorrectionVectorRotation()
{
    return correctionVectorRotation;
}
 
const std::vector<InputProvider::StreamFrameInfo> VulkanWrapper::getFramesInfo(int view)
{
    if (requestChange) {
        context.device.waitIdle();
#ifdef HVT_UDP_CONTROL
        mutexAct.lock();
#endif
        drawing.cleanup();
        renderPass->cleanUp();
        renderPass->init(inputProvider);
        drawing.init(inputProvider);
        requestChange = false;
#ifdef HVT_UDP_CONTROL
        mutexAct.unlock();
#endif
    }
    inputProvider->acquireStreamsFrames(params.getVirtualExtrinsics(view), streamFrameInfos);
    if (requireAverageInputPosition) {
        setAverageInputPosition(streamFrameInfos);
    }
 
    assert(correctionVectorRotation.size() == streamFrameInfos.size());
 
    for (int i = 0; i < correctionVectorRotation.size(); i++) {
        streamFrameInfos[i].extrinsics.rotation += correctionVectorRotation[i];
        streamFrameInfos[i].extrinsics.position += correctionVectorTranslation[i];
 
        if (streamFrameInfos[i].intrinsics.index() == 0) {
            std::get<InputProvider::PerspectiveIntrinsics>(streamFrameInfos[i].intrinsics).focals += correctionFocal[i];
            std::get<InputProvider::PerspectiveIntrinsics>(streamFrameInfos[i].intrinsics).principle += correctionPrincipalPt[i];
        }
        
    }
 
    return streamFrameInfos;
}
 
 
 
#ifdef USE_OPENXR
XrGraphicsBindingVulkan2KHR VulkanWrapper::getGraphicsBinding() {
    XrGraphicsBindingVulkan2KHR m_graphicsBinding{XR_TYPE_GRAPHICS_BINDING_VULKAN2_KHR};
    m_graphicsBinding.instance = context.instance;
    m_graphicsBinding.physicalDevice = context.physicalDevice;
    m_graphicsBinding.device = context.device;
    m_graphicsBinding.queueFamilyIndex = context.queueFamilyIndex;
    m_graphicsBinding.queueIndex = 0;
    return m_graphicsBinding;
}
 
 
void VulkanWrapper::endCleanUp() {
    if(context.isInitialized()){
        context.cleanUp();
    }
}
 
#endif
 
void VulkanWrapper::updateSpaceTransform(glm::vec3 translation, glm::vec3 rotation, glm::vec4 fov, int view) {
 
    params.updateSpaceTransform(translation, rotation, fov, view);
}