HoviTron Video Pipeline: RaytrixStreamer Class Reference

Inheritance diagram for RaytrixStreamer:

Public Member Functions
	RaytrixStreamer (const uint8_t uuid[VK_UUID_SIZE])
	Launch all init functions defined in RaytrixStreamer class.

	~RaytrixStreamer ()
	Free all the memories that were allocated during the streaming process.

void	enumerateStreamsParameters (uint32_t streamsCount, HvtRGBDStreamParameters parameters) const

void	importStreamImages (const HvtStreamImagesExportInfo &exportInfos)

void	importSemaphore (const HvtSemaphoreExportInfo &exportInfos)

void	destroySemaphore (Semaphore *sem) const

void	startStreaming ()

void	acquireStreamsFrames (const HvtAcquireStreamFramesInfo &infos)

void	releaseStreamsFrames (Semaphore *waitSem)

void	stopStreaming ()

float	getMaxDepth ()
	Get the maximum depth.

float	getMinDepth ()
	Get the minimum depth.

HvtIntrinsicsPerspective	getRxIntrinsicParams (size_t idxCam)
	Get the intrinsic parameters of the camera of ID "idxCam". The intrinsic parameters are found through the Raytrix SDK.

HvtExtrinsics	getRxExtrinsicParams (size_t idxCam)
	Get the extrinsic parameters of the camera of ID "idxCam". The extrinsic parameters are found through the Raytrix SDK and the extrinsic file.

Rx::LFR::CImageQueue &	getCamBuffer (size_t idxCam)
	Get the camera buffer in the list of camera buffers.

	RaytrixStreamer (const uint8_t uuid[VK_UUID_SIZE])
	Launch all init functions defined in RaytrixStreamer class.

	~RaytrixStreamer ()
	Free all the memories that were allocated during the streaming process.

void	enumerateStreamsParameters (uint32_t streamsCount, HvtRGBDStreamParameters parameters) const

void	importStreamImages (const HvtStreamImagesExportInfo &exportInfos)

void	importSemaphore (const HvtSemaphoreExportInfo &exportInfos)

void	destroySemaphore (Semaphore *sem) const

void	startStreaming ()

void	acquireStreamsFrames (const HvtAcquireStreamFramesInfo &infos)

void	releaseStreamsFrames (Semaphore *waitSem)

void	stopStreaming ()

float	getMaxDepth ()
	Get the maximum depth.

float	getMinDepth ()
	Get the minimum depth.

HvtIntrinsicsPerspective	getRxIntrinsicParams (size_t idxCam)
	Get the intrinsic parameters of the camera of ID "idxCam". The intrinsic parameters are found through the Raytrix SDK.

HvtExtrinsics	getRxExtrinsicParams (size_t idxCam)
	Get the extrinsic parameters of the camera of ID "idxCam". The extrinsic parameters are found through the Raytrix SDK and the extrinsic file.

Rx::LFR::CImageQueue &	getCamBuffer (size_t idxCam)
	Get the camera buffer in the list of camera buffers.

Public Member Functions inherited from Handle< RaytrixStreamer, HvtStreamingContext, HvtHandleType::CONTEXT >
HvtStreamingContext	to_handle ()

HvtStreamingContext	to_handle ()

HvtStreamingContext	to_handle ()

HvtStreamingContext	to_handle ()

Additional Inherited Members
Static Public Member Functions inherited from Handle< RaytrixStreamer, HvtStreamingContext, HvtHandleType::CONTEXT >
static RaytrixStreamer *	check (HvtStreamingContext hvtHandle)

static RaytrixStreamer *	check (HvtStreamingContext hvtHandle)

static RaytrixStreamer *	check (HvtStreamingContext hvtHandle)

static RaytrixStreamer *	check (HvtStreamingContext hvtHandle)

static RaytrixStreamer *	opt_check (HvtStreamingContext hvtHandle)

static RaytrixStreamer *	opt_check (HvtStreamingContext hvtHandle)

static RaytrixStreamer *	opt_check (HvtStreamingContext hvtHandle)

static RaytrixStreamer *	opt_check (HvtStreamingContext hvtHandle)

Static Public Attributes inherited from Handle< RaytrixStreamer, HvtStreamingContext, HvtHandleType::CONTEXT >
static constexpr HvtHandleType	staticType

Detailed Description

Definition at line 122 of file raytrixdll-v5/RaytrixStreamer.h.

Constructor & Destructor Documentation

◆ RaytrixStreamer() [1/2]

RaytrixStreamer::RaytrixStreamer ( const uint8_t uuid[VK_UUID_SIZE] )

Launch all init functions defined in RaytrixStreamer class.

Parameters

uuid	The UUID of the vulkan GPU.

Definition at line 1042 of file raytrixdll-v5/RaytrixStreamer.cpp.

{
    findVulkanGPU(uuid);
    readInfoFromJSON();
    initRxSDK();
    if (infoJSON.loadFromRawFile) {
        initRxRayFile();
    }
    else {
        initRxCams();
    }
    initRxCudaCompute();
    initStreamParameters();
    initArrayCuda();
    initMutex();
}

◆ ~RaytrixStreamer()

RaytrixStreamer::~RaytrixStreamer ( )

Free all the memories that were allocated during the streaming process.

Definition at line 1059 of file raytrixdll-v5/RaytrixStreamer.cpp.

{
    std::cout << "---- Destroy RaytrixStreamer dll ----" << std::endl;
    if (synthesisSem != NULL)
        delete synthesisSem;
    for (size_t i = 0; i < infoJSON.numCams; ++i) {
        xCamServer.GetCamera(i).Close();
    }
    if (infoJSON.loadFromRawFile) {
        delete[] seqReaders;
        delete[] rayImages;
    }
    Rx::LFR::CLightFieldRuntime::End();
}

◆ RaytrixStreamer() [2/2]

RaytrixStreamer::RaytrixStreamer ( const uint8_t uuid[VK_UUID_SIZE] )

Launch all init functions defined in RaytrixStreamer class.

Parameters

uuid	The UUID of the vulkan GPU.

Member Function Documentation

◆ acquireStreamsFrames()

void RaytrixStreamer::acquireStreamsFrames ( const HvtAcquireStreamFramesInfo & infos )

Definition at line 1242 of file raytrixdll-v5/RaytrixStreamer.cpp.

{
    gpuErrchk(cudaSetDevice(vulkanGPU));
    if (infos.frameInfoCount != readStreams.size())
    {
        throw HvtResult::HVT_ERROR_WRONG_BUFFER_SIZE;
    }
 
    swapPendingToReading();
 
    auto imageIndex = slotReadingIndex;
 
    for (uint32_t i = 0; i < infos.frameInfoCount; ++i)
    {
        gpuErrchk(cudaEventSynchronize(cudaEvents[i]));
 
        auto& stream = readStreams.at(i);
        auto& desc = infos.pStreamFrameInfos[i];
        desc.extrinsics = stream.extrinsics;
        desc.intrinsics = stream.intrinsics;
        desc.imageIndex = imageIndex;
    }
}

◆ destroySemaphore()

void RaytrixStreamer::destroySemaphore ( Semaphore * sem ) const

Definition at line 1217 of file raytrixdll-v5/RaytrixStreamer.cpp.

{
    cudaSetDevice(vulkanGPU);
    delete sem;
}

◆ enumerateStreamsParameters()

void RaytrixStreamer::enumerateStreamsParameters	(	uint32_t *	streamsCount,
		HvtRGBDStreamParameters *	parameters
	)		const

Definition at line 1075 of file raytrixdll-v5/RaytrixStreamer.cpp.

{
    cudaSetDevice(vulkanGPU);
    if (!parameters)
    {
        *streamsCount = readStreams.size();
        return;
    }
 
    if (*streamsCount != readStreams.size())
    {
        throw HvtResult::HVT_ERROR_WRONG_BUFFER_SIZE;
    }
 
    int i = 0;
    for (const auto& readStream : readStreams)
    {
        HvtRGBDStreamParameters* SP = new HvtRGBDStreamParameters;
        SP->colorResolution = { (uint32_t)readStream.width_color, (uint32_t)readStream.height_color };
        SP->depthResolution = { (uint32_t)readStream.width_depth, (uint32_t)readStream.height_depth };
        SP->nearDepth = readStream.anear;
        SP->farDepth = readStream.afar;
        SP->colorFormat = (HvtImageFormat)readStream.colorFormat;
        SP->depthFormat = (HvtImageFormat)readStream.depthFormat;
        SP->slotCount = numSlots;
        SP->projectionType = readStream.projectionType;
 
        parameters[i] = *SP;
        //snprintf(parameters->name, HVT_MAX_STREAM_NAME_LENGHT, "%s", deviceNames.at(i).c_str());
        ++i;
    }
}

◆ getCamBuffer() [1/2]

Rx::LFR::CImageQueue & RaytrixStreamer::getCamBuffer ( size_t idxCam )

Get the camera buffer in the list of camera buffers.

Parameters

idxCam ID of the buffer.

Returns: A copy of the buffer.

Definition at line 614 of file raytrixdll-v5/RaytrixStreamer.cpp.

{
    return camBuffers[idxCam];
}

◆ getCamBuffer() [2/2]

Rx::LFR::CImageQueue & RaytrixStreamer::getCamBuffer ( size_t idxCam )

Get the camera buffer in the list of camera buffers.

Parameters

idxCam ID of the buffer.

Returns: A copy of the buffer.

◆ getMaxDepth() [1/2]

float RaytrixStreamer::getMaxDepth ( )

Get the maximum depth.

Returns: maximum depth.

Definition at line 533 of file raytrixdll-v5/RaytrixStreamer.cpp.

                                   {
    const float ZFar = 1.3f; // -> !!! MAGIC NUMBER !!! from AcquKirt -> not used here
    return ZFar;
}

◆ getMaxDepth() [2/2]

float RaytrixStreamer::getMaxDepth ( )

Get the maximum depth.

Returns: maximum depth.

◆ getMinDepth() [1/2]

float RaytrixStreamer::getMinDepth ( )

Get the minimum depth.

Returns: minimum depth.

Definition at line 538 of file raytrixdll-v5/RaytrixStreamer.cpp.

                                   {
    const float ZNear = 0.4f; // -> !!! MAGIC NUMBER !!! from AcquKirt -> not used here
    return ZNear;
}

◆ getMinDepth() [2/2]

float RaytrixStreamer::getMinDepth ( )

Get the minimum depth.

Returns: minimum depth.

◆ getRxExtrinsicParams() [1/2]

HvtExtrinsics RaytrixStreamer::getRxExtrinsicParams ( size_t idxCam )

Get the extrinsic parameters of the camera of ID "idxCam". The extrinsic parameters are found through the Raytrix SDK and the extrinsic file.

Parameters

idxCam ID of the camera.

Returns: The extrinsic params through the structure HvtExtrinsics that is used by the Hovitron pipeline.

Definition at line 558 of file raytrixdll-v5/RaytrixStreamer.cpp.

                                                                 {
    HvtPosition pos;
    HvtRotation rot;
    unsigned isExCalib;
    Rx::LFR::CParameters<Rx::LFR::Params::ECalib::ID>& paramsCal = camCalibrations[idxCam]->GetParams();
    paramsCal.GetValue(Rx::LFR::Params::ECalib::IsExtrinsicCalibrated, isExCalib);
    if (isExCalib != 1) {
        std::cerr << "Camera " << idxCam << " is not extrinsincly calibrated... Reconstruction must be wrong." << std::endl;
    }
    Rx::CRxArrayDouble trans;
    paramsCal.GetValue(Rx::LFR::Params::ECalib::Translation_Global_Sensor, trans);
    double* dataPtr = (double*)trans.GetPointer();
    size_t sizeData = trans.Length();
    bool checkExParams = true;
    if (sizeData < 1)
        checkExParams = false;
 
    std::ifstream jsonRead{ infoJSON.extrinsicParamsPath, std::ifstream::binary };
    if (!jsonRead.good()) {
        std::cout << "Impossible to read JSON extrinsic file, check path... Path given: " << infoJSON.extrinsicParamsPath << std::endl;
        throw HVT_ERROR_NOT_FOUND;
    }
    nlohmann::json jsonContent;
    jsonRead >> jsonContent;
 
    if (checkExParams) {
        pos.x = (float)dataPtr[2] + (double) jsonContent["cameras"][idxCam]["Position"][0]; // x is forward/backward
        pos.y = (float)dataPtr[0] + (double) jsonContent["cameras"][idxCam]["Position"][1]; // y is left/right
        pos.z = (float)dataPtr[1] + (double) jsonContent["cameras"][idxCam]["Position"][2]; // z is up/down
        referencePlaneToCameraPlane[idxCam] = pos.x;
        double data;
        paramsCal.GetValue(Rx::LFR::Params::ECalib::Rotation_Global_Sensor_Pitch, data);
        rot.pitch = (float)data + (double) jsonContent["cameras"][idxCam]["Rotation"][0];
        paramsCal.GetValue(Rx::LFR::Params::ECalib::Rotation_Global_Sensor_Roll, data);
        rot.roll = (float)data + (double) jsonContent["cameras"][idxCam]["Rotation"][1];
        paramsCal.GetValue(Rx::LFR::Params::ECalib::Rotation_Global_Sensor_Yaw, data);
        rot.yaw = (float)data + (double) jsonContent["cameras"][idxCam]["Rotation"][2];
    }
    else {
        pos.x = 0.0f;
        pos.y = 0.0f;
        pos.z = 0.0f;
        referencePlaneToCameraPlane[idxCam] = 1.0f;
        rot.pitch = 0.0f;
        rot.roll = 0.0f;
        rot.yaw = 0.0f;
    }
    jsonRead.close();
 
 
    HvtExtrinsics extrinsics;
    extrinsics.position = pos;
    extrinsics.rotation = rot;
    return extrinsics;
}

◆ getRxExtrinsicParams() [2/2]

HvtExtrinsics RaytrixStreamer::getRxExtrinsicParams ( size_t idxCam )

Get the extrinsic parameters of the camera of ID "idxCam". The extrinsic parameters are found through the Raytrix SDK and the extrinsic file.

Parameters

idxCam ID of the camera.

Returns: The extrinsic params through the structure HvtExtrinsics that is used by the Hovitron pipeline.

◆ getRxIntrinsicParams() [1/2]

HvtIntrinsicsPerspective RaytrixStreamer::getRxIntrinsicParams ( size_t idxCam )

Get the intrinsic parameters of the camera of ID "idxCam". The intrinsic parameters are found through the Raytrix SDK.

Parameters

idxCam ID of the camera.

Returns: The intrinsic params through the structure HvtIntrinsicsPerspective that is used by the Hovitron pipeline.

Definition at line 543 of file raytrixdll-v5/RaytrixStreamer.cpp.

                                                                            {
    Rx::LFR::CParameters<Rx::LFR::Params::ECalib::ID>& paramsCal = camCalibrations[idxCam]->GetParams();
    double f, sizePixel;
    paramsCal.GetValue(Rx::LFR::Params::ECalib::MainLensThick_NominalFocalLengthMM, f);
    paramsCal.GetValue(Rx::LFR::Params::ECalib::Sensor_PhysicalPixelSizeMM, sizePixel);
 
    HvtIntrinsicsPerspective intrinsicParams;
    intrinsicParams.focalX = (float) f / (((float)widthDefault / (float)infoJSON.widthDepth) * sizePixel);
    intrinsicParams.focalY = (float) f / (((float)heightDefault / (float)infoJSON.heightDepth) * sizePixel);
    intrinsicParams.principlePointX = ((float)infoJSON.width) / 2.0f;
    intrinsicParams.principlePointY = ((float)infoJSON.height) / 2.0f;
 
    return intrinsicParams;
}

◆ getRxIntrinsicParams() [2/2]

HvtIntrinsicsPerspective RaytrixStreamer::getRxIntrinsicParams ( size_t idxCam )

Get the intrinsic parameters of the camera of ID "idxCam". The intrinsic parameters are found through the Raytrix SDK.

Parameters

idxCam ID of the camera.

Returns: The intrinsic params through the structure HvtIntrinsicsPerspective that is used by the Hovitron pipeline.

◆ importSemaphore()

void RaytrixStreamer::importSemaphore ( const HvtSemaphoreExportInfo & exportInfos )

Definition at line 1184 of file raytrixdll-v5/RaytrixStreamer.cpp.

{
    cudaSetDevice(vulkanGPU);
    if (exportInfos.semaphore == NULL)
    {
        synthesisSem->isValid = false;
        synthesisSem->sem = NULL;
        return;
    }
 
    cudaExternalSemaphoreHandleDesc externalSemaphoreHandleDesc = {};
 
    if (exportInfos.type & HvtSemaphoreType::HVT_SEMAPHORE_TYPE_OPAQUE_WIN32_BIT)
    {
        externalSemaphoreHandleDesc.type = cudaExternalSemaphoreHandleTypeOpaqueWin32;
    }
    else if (exportInfos.type & HvtSemaphoreType::HVT_SEMAPHORE_TYPE_OPAQUE_WIN32_KMT_BIT)
    {
        externalSemaphoreHandleDesc.type = cudaExternalSemaphoreHandleTypeOpaqueWin32Kmt;
    }
    else if (exportInfos.type & HvtSemaphoreType::HVT_SEMAPHORE_TYPE_SYNC_FD_BIT)
    {
        externalSemaphoreHandleDesc.type = cudaExternalSemaphoreHandleTypeOpaqueFd;
    }
 
    externalSemaphoreHandleDesc.handle.win32.handle = (HANDLE)exportInfos.semaphore;
    externalSemaphoreHandleDesc.flags = 0;
 
    cudaImportExternalSemaphore(&synthesisSem->sem, &externalSemaphoreHandleDesc);
    synthesisSem->isValid = true;
    // cudaWaitExternalSemaphoresAsync();
}

◆ importStreamImages()

void RaytrixStreamer::importStreamImages ( const HvtStreamImagesExportInfo & exportInfos )

Definition at line 1108 of file raytrixdll-v5/RaytrixStreamer.cpp.

{
    cudaSetDevice(vulkanGPU);
    auto isDepth = (bool)exportInfos.depth;
    auto& stream = readStreams.at(exportInfos.streamIndex);
    //auto format = isDepth ? stream.depthFormat : stream.colorFormat;
    // auto size = isDepth ? stream.depthFrameSize : stream.colorFrameSize;
    auto& slots = isDepth ? stream.depthSlots : stream.colorSlots;
    auto mipLevels = 1;
 
    // std::cout << "Init memory buffers CUDA" << std::endl;
 
    if (exportInfos.imagesCount != numSlots)
    {
        throw HvtResult::HVT_ERROR_WRONG_BUFFER_SIZE;
    }
 
    for (int i = 0; i < numSlots; ++i)
    {
        auto mem = exportInfos.pImages[i];
        // std::cout << "External Memory loop" << std::endl;
 
        cudaExternalMemory_t cudaExtMemImageBuffer;
        cudaMipmappedArray_t cudaMipmappedImageArray;
 
        cudaExternalMemoryHandleDesc cudaExtMemHandleDesc;
        memset(&cudaExtMemHandleDesc, 0, sizeof(cudaExtMemHandleDesc));
 
        cudaExtMemHandleDesc.type = cudaExternalMemoryHandleTypeOpaqueWin32;
        cudaExtMemHandleDesc.handle.win32.handle = (HANDLE)mem.handle;
        cudaExtMemHandleDesc.size = mem.size;
 
        gpuErrchk(cudaImportExternalMemory(&cudaExtMemImageBuffer, &cudaExtMemHandleDesc));
 
        cudaExternalMemoryMipmappedArrayDesc externalMemoryMipmappedArrayDesc;
        memset(&externalMemoryMipmappedArrayDesc, 0, sizeof(externalMemoryMipmappedArrayDesc));
 
        cudaExtent extent = isDepth ? make_cudaExtent(stream.width_depth, stream.height_depth, 0) : make_cudaExtent(stream.width_color, stream.height_color, 0);
        cudaChannelFormatDesc formatDesc;
        formatDesc.x = isDepth ? 32 : 8;
        formatDesc.y = isDepth ? 0 : 8;
        formatDesc.z = isDepth ? 0 : 8;
        formatDesc.w = isDepth ? 0 : 8;
        formatDesc.f = isDepth ? cudaChannelFormatKindFloat : cudaChannelFormatKindUnsigned;
 
        externalMemoryMipmappedArrayDesc.offset = 0;
        externalMemoryMipmappedArrayDesc.formatDesc = formatDesc;
        externalMemoryMipmappedArrayDesc.extent = extent;
        externalMemoryMipmappedArrayDesc.flags = 0;
        externalMemoryMipmappedArrayDesc.numLevels = mipLevels;
 
        gpuErrchk(cudaExternalMemoryGetMappedMipmappedArray(&cudaMipmappedImageArray, cudaExtMemImageBuffer, &externalMemoryMipmappedArrayDesc));
 
        cudaArray_t cudaMipLevelArray;
        cudaResourceDesc resourceDesc;
 
        gpuErrchk(cudaGetMipmappedArrayLevel(&cudaMipLevelArray, cudaMipmappedImageArray, 0));
        // cudaMemcpy2DArrayToArray(cudaMipLevelArray, 0, 0, cudaMipLevelArray, 0, 0, stream.width * sizeof(uchar4), stream.height, cudaMemcpyDeviceToDevice);
 
        memset(&resourceDesc, 0, sizeof(resourceDesc));
        resourceDesc.resType = cudaResourceTypeArray;
        resourceDesc.res.array.array = cudaMipLevelArray;
 
        cudaSurfaceObject_t surfaceObject;
        gpuErrchk(cudaCreateSurfaceObject(&surfaceObject, &resourceDesc));
 
        ImageSlot IS;
        IS.size = mem.size;
        IS.vk_handle = &mem.handle;
        IS.cuda_ptr_surf = surfaceObject;
 
        slots.at(i) = IS;
    }
    (isDepth ? stream.importedDepth : stream.importedColor) = true;
}

◆ releaseStreamsFrames()

void RaytrixStreamer::releaseStreamsFrames ( Semaphore * waitSem )

Definition at line 1266 of file raytrixdll-v5/RaytrixStreamer.cpp.

{
    std::cout << "release frames" << std::endl;
}

◆ startStreaming()

void RaytrixStreamer::startStreaming ( )

Definition at line 1223 of file raytrixdll-v5/RaytrixStreamer.cpp.

{
    //cudaSetDevice(vulkanGPU);
    // assert that we have everything ready for streaming
    for (auto& stream : readStreams)
    {
        if (!stream.importedColor || !stream.importedDepth)
        {
            throw HvtResult::HVT_ERROR_CALL_ORDER;
        }
    }
 
    // Start worker
    running = true;
    std::cout << "Starting streaming" << std::endl;
    streamingThread = std::thread([&]
        { streamingLoop(); });
}

◆ stopStreaming()

void RaytrixStreamer::stopStreaming ( )

Definition at line 1271 of file raytrixdll-v5/RaytrixStreamer.cpp.

{
    cudaSetDevice(vulkanGPU);
    running = false;
    if (streamingThread.joinable())
    {
        streamingThread.join();
    }
}

The documentation for this class was generated from the following files:

Public Member Functions

Additional Inherited Members

Detailed Description

Constructor & Destructor Documentation

◆ RaytrixStreamer() [1/2]

◆ ~RaytrixStreamer()

◆ RaytrixStreamer() [2/2]

Member Function Documentation

◆ acquireStreamsFrames()

◆ destroySemaphore()

◆ enumerateStreamsParameters()

◆ getCamBuffer() [1/2]

◆ getCamBuffer() [2/2]

◆ getMaxDepth() [1/2]

◆ getMaxDepth() [2/2]

◆ getMinDepth() [1/2]

◆ getMinDepth() [2/2]

◆ getRxExtrinsicParams() [1/2]

◆ getRxExtrinsicParams() [2/2]

◆ getRxIntrinsicParams() [1/2]

◆ getRxIntrinsicParams() [2/2]

◆ importSemaphore()

◆ importStreamImages()

◆ releaseStreamsFrames()

◆ startStreaming()

◆ stopStreaming()