HoviTron Video Pipeline
raytrixdll-v6/RaytrixStreamer.h
1#pragma once
2/* ----------------------
3* Copyright 2023 Université Libre de Bruxelles(ULB), Universidad Politécnica de Madrid(UPM), CREAL, Deutsches Zentrum für Luft - und Raumfahrt(DLR)
4
5* Licensed under the Apache License, Version 2.0 (the "License");
6* you may not use this file except in compliance with the License.
7* You may obtain a copy of the License at < http://www.apache.org/licenses/LICENSE-2.0%3E
8
9* Unless required by applicable law or agreed to in writing, software
10* distributed under the License is distributed on an "AS IS" BASIS,
11* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12* See the License for the specific language governing permissionsand
13* limitations under the License.
14---------------------- */
15
16// CREAL - HOVITRON Pipeline header
17#ifdef _WIN32
18#include <filesystem>
19namespace fs = std::filesystem;
20#include <Windows.h>
21#define HVT_ENTRY_ATTR __declspec(dllexport)
22#else
23#include <experimental/filesystem>
24namespace fs = std::experimental::filesystem;
25typedef void* HANDLE;
26#endif
27
28#define HVT_FUNCTION_PROTOTYPES
29#include "HvtStreamingAPI.h"
30
31// C/C++ header
32#include <conio.h>
33#include <iostream>
34#include <thread>
35#include <mutex>
36#include <atomic>
37#include <array>
38#include <fstream>
39
40// Raytrix Header
41#include "Rx.Core\RxException.h"
42#include "Rx.LFR/LightFieldRuntime.h"
43#include "Rx.LFR/CameraServer.h"
44#include "Rx.LFR/CalibrationManager.h"
45#include "Rx.LFR/Cuda.h"
46#include "Rx.LFR/CudaCompute.h"
47#include "Rx.LFR/ImageQueue.h"
48#include "Rx.LFR\RayFileReader.h"
49#include "Rx.Core/EColorMap.h"
50#include "Rx.LFR/RayFileWriter.h"
51#include "Rx.LFR/SeqFileReader.h"
52#include "Rx.LFR/SeqFileWriter.h"
53
54// Cuda header
55#include <cuda.h>
56#include <cuda_runtime.h>
57
58// RaytrixStreamer header
59#include "types.h"
60#include "KernelCuda.cuh"
61#include "nlohmann/json.hpp"
62
63#ifndef VK_UUID_SIZE
64#define VK_UUID_SIZE 16U
65#endif
66
67enum class HvtHandleType
68{
69 CONTEXT,
70 SEMAPHORE
71};
72
73template <typename OurType, typename HvtHandle, HvtHandleType htype>
74struct Handle
75{
76 Handle() : type(htype) {}
77 static constexpr HvtHandleType staticType = htype;
78
79 static OurType* check(HvtHandle hvtHandle)
80 {
81 Handle* h = reinterpret_cast<Handle*>(hvtHandle);
82 if (h && staticType == h->type)
83 {
84 return static_cast<OurType*>(h);
85 }
86 else
87 {
88 throw HvtResult::HVT_ERROR_INVALID_HANDLE;
89 }
90 }
91
92 static OurType* opt_check(HvtHandle hvtHandle)
93 {
94 if (!hvtHandle)
95 {
96 return nullptr;
97 }
98 return check(hvtHandle);
99 }
100
101 HvtHandle to_handle()
102 {
103 return reinterpret_cast<HvtHandle>(this);
104 }
105
106private:
107 HvtHandleType type;
108};
109
110struct Semaphore : public Handle<Semaphore, HvtSemaphore, HvtHandleType::SEMAPHORE>
111{
112 cudaExternalSemaphore_t sem;
113 bool isValid;
114};
115
116class RaytrixStreamer : public Handle<RaytrixStreamer, HvtStreamingContext, HvtHandleType::CONTEXT>
117{
118 public:
119 /*-------------- Constructor/Destructor */
124 RaytrixStreamer(const uint8_t uuid[VK_UUID_SIZE]);
129 /*--------------------------------------*/
130
131 /*---------- Hovitron API C entry points*/
132 void enumerateStreamsParameters(uint32_t* streamsCount, HvtRGBDStreamParameters* parameters) const;
133 void importStreamImages(const HvtStreamImagesExportInfo& exportInfos);
134 void importSemaphore(const HvtSemaphoreExportInfo& exportInfos);
135 void destroySemaphore(Semaphore* sem) const;
136 void startStreaming();
137 void acquireStreamsFrames(const HvtAcquireStreamFramesInfo& infos);
138 void releaseStreamsFrames(Semaphore* waitSem);
139 void stopStreaming();
140 /*--------------------------------------*/
141
142 /*--------- Getter*/
147 float getMaxDepth();
152 float getMinDepth();
170 Rx::LFR::CImageQueue& getCamBuffer(size_t idxCam);
171 /*---------------*/
172
173private:
174 static constexpr auto numSlots = 3;
175 using Clock = std::chrono::steady_clock;
176
177 /*---------- Hovitron API structure*/
178 // Structure for an image slot. The image is given to the vulkan handle through the cudaSurfaceObject_t.
179 struct ImageSlot
180 {
181 cudaSurfaceObject_t cuda_ptr_surf;
182 HVT_OPAQUE_HANDLE* vk_handle;
183 int32_t size;
184 };
185 // A stream is a structure that is sent to the Hovitron Pipeline. It gives all necessery information of the camera and the image.
186 struct ReadStream
187 {
188 size_t colorFrameStride;
189 size_t colorFrameSize;
190 size_t depthFrameStride;
191 size_t depthFrameSize;
192
193 HvtProjectionType projectionType;
194 HvtIntrinsics intrinsics;
195 HvtExtrinsics extrinsics;
196
197 HvtImageFormat colorFormat;
198 HvtImageFormat depthFormat;
199
200 int width_color;
201 int height_color;
202 int width_depth;
203 int height_depth;
204 float anear;
205 float afar;
206
207 std::array<ImageSlot, numSlots> colorSlots;
208 std::array<ImageSlot, numSlots> depthSlots;
209 bool importedColor = false;
210 bool importedDepth = false;
211
212 Clock::duration framePeriod;
213 int frameCount = 1;
214 int streamedFrame = -1;
215
221 int frameIndex(Clock::duration time) const;
227 bool nextFrameReady(Clock::duration time) const;
228 };
229 /*---------------------------------*/
230
231 /*-------------- RaytrixStreamer DLL structure*/
232 // Contain all the information about the way the DLL should work.
233 struct infoJSON_t {
234 // Number of cams
235 size_t numCams = 0;
236 //If the DLL should force the use of a specific RGB resolution
237 bool forceRGBResolution = false;
238 //If the DLL should force the use of a specific RGB depth
239 bool forceDepthResolution = false;
240 // Width of the RGB image
241 unsigned int width = 0;
242 // Height of the RGB image
243 unsigned int height = 0;
244 // Width of the Depth image
245 unsigned int widthDepth = 0;
246 // Height of the Depth image
247 unsigned int heightDepth = 0;
248 // The ID of the cameras
249 unsigned int* uIDs = nullptr;
250 // The filename of the compute parameters
251 Rx::CRxString filenameComputeParameters = Rx::CRxString();
252 // True, if we use sequence files
253 bool loadFromRawFile = false;
254 // Path of these seq files
255 Rx::CRxString* filenameRawFiles = nullptr;
256 // True, if a or some specific calibrations is(are) wanted
257 bool specificCal = false;
258 // The ID of the calibrations
259 Rx::CRxString* GUIDsCalib = nullptr;
260 // The exposure of the cameras to set (allow of better and faster depth computation)
261 float* exposureCams = nullptr;
262 // The path of the extrinsic camera parameters
263 std::string extrinsicParamsPath = std::string();
264 };
265 /*--------------------------------------------*/
266
267 /*--------- Hovitron API functions*/
268
276 void streamingLoop();
282 void uploadFrame(size_t streamId, ReadStream& stream);
286 void swapStreamingToPending();
290 void swapPendingToReading();
291 /*--------------------------------*/
292
293 /*------- Init Raytrix DLL functions */
294
298 void readInfoFromJSON();
302 void initRxCams();
307 void findVulkanGPU(const uint8_t uuid[VK_UUID_SIZE]);
311 void initStreamParameters();
315 void initRxCudaCompute();
319 void initRxSDK();
323 void initArrayCuda();
327 void initRxRayFile();
331 void initMutex();
332 /*---------------------------------*/
333
334 /*--------------- RaytrixDLL functions for RGBD acquire and refine tasks*/
335
339 void acquireRGBD();
345 void refineRGBD();
349 void getRGBDFormats();
354 void launchAsyncLoader(bool needToWaitthreadToFinish = false);
359 void asyncLoaderImage();
364 void asyncLoaderImageOneCam(size_t idxCam);
368 void asyncLoaderImageFromSeqFiles();
373 void asyncLoaderImageFromOneSeqFile(size_t idxCam);
374 // A list of mutexex to assure that we do not have race conditions when the buffer image is moved to the CRxImage object.
375 std::mutex* loaderMtxs;
376 // A list of condition variables that wait until the condition is verified.
377 std::condition_variable* loaderCVs;
378 // The list of conditions to block/unblock load of new data.
379 bool* canLoadImgs;
380 // A mutex to assure that we do not have race conditions between the syncrhonization of cameras and the copy of the data in our CUDA arrays.
381 std::mutex syncMtx;
382 // The condition variable that waits until the condition is verified.
383 std::condition_variable syncCV;
384 // The condition to block/unblock the copy of the datas.
385 bool imgsLoaded = false;
386 /*----------------------------------------------------------------------*/
387
388 /*---------- Debug functions*/
389
401 template<typename T> void writeInFileWindow(T* dataCorrect, T* dataDebug, size_t width, size_t height, size_t sizeWindow, int numCam = 0);
406 void debugFloatDepth();
407 /*--------------------------*/
408
409 /*--------- Hovitron API variables*/
410 int vulkanGPU;
411 Semaphore* synthesisSem;
412 std::vector<ReadStream> readStreams;
413 std::atomic_bool running = false;
414 std::thread streamingThread;
415 uint32_t slotReadingIndex = 0;
416 uint32_t slotPendingIndex = 1;
417 uint32_t slotStreamingIndex = 2;
418 std::atomic_bool newDataInPending = false;
419 std::mutex indicesMutex;
420 std::mutex queueMutex;
421 float numFrames = 0.0f;
422 /*--------------------------------*/
423
424 /*----------------- RaytrixStreamer variables*/
425 // CUDA Streams for stream concurrency programming
426 cudaStream_t* cudaStreams;
427 // CUDA Event to synchronize a stream
428 cudaEvent_t* cudaEvents;
429 // All information that the DLL needs
430 infoJSON_t infoJSON;
431 // A list of pointer to camera calibrations
432 Rx::LFR::CCalibration** camCalibrations;
433 // The camera server class that is able to find attached cameras.
434 Rx::LFR::CCameraServer xCamServer;
435 // This is the CUDA compute instances
436 Rx::LFR::CCudaCompute* xCudaComputes;
437 // Two lists of USHORT arrays
438 unsigned short** imgRGB, ** imgDepth;
439 // Two lists of UCHAR/UBYTE arrays (only RGB image should be in UChar)
440 uchar** finalRGB, ** imgUByteRGB;
441 // Three lists of float arrays (only depth image should be in Float)
442 float** finalDepth, ** imgFloatDepth, ** prevFinalDepth;;
443 // A list of camera buffers
444 Rx::LFR::CImageQueue* camBuffers;
445 // Capture mode of the cameras
446 const Rx::ECamTriggerMode camCaptureMode = Rx::ECamTriggerMode::Camera_FreeRun;
447 // Set up the image buffer properties
448 const unsigned uBufferSize = 1;
449 // Set up the image buffer properties
450 const bool bOverwrite = true;
451 // List of CRxImage that are used as a tmp image loader (for camera acquired method)
452 Rx::CRxImage* capturedImages;
453 // List of the format of the RGB and depth images
454 Rx::CRxImageFormat* imgFormatRGB, * imgFormatDepth;
455 // Cuda image variable to extract the cudaRGB and cudaDepth from the RxSDK
456 Rx::LFR::CudaImage* cuImgs, * cuDepths;
457 // A boolean to know if the formats have been already loaded
458 bool isFormatLoaded = false;
459 // A list of CRayImage that are used as a tmp image loader (for Sequence file acquired method)
460 Rx::LFR::CRayImage* rayImages;
461 // A list of sequence files
462 Rx::LFR::CSeqFileReader* seqReaders;
463 // The pitch of the CUDA RGB and depth array, pitch = number of bytes in row <=> IdxFromPitch = pitch / size(T)
464 size_t* pitchInCudaRGB, * pitchInCudaDepth;
465 // List of reference plane position, allows to where where the reference plane has been put from one camera
466 float* referencePlaneToCameraPlane;
467 // Size of raytrix by default
468 size_t widthDefault = 3840, heightDefault = 2160;
469 // Channel of depth in Depth3D array
470 const size_t channelDepth = 2;
471 // Max time to capture imgs between cams (in ??)
472 const double MAX_TIME_BETWEEN_CAMS = 1000.0;
473 // Max time to wait for buffer to be filled (in ms)
474 const int MAX_TIME_WAIT_BUFFER = 1000;
475 // Name of the environment variable allows loading the parameters
476 const char* ENV_NAME = "HVT_RXDLL_PARAMS_PATH";
477 // ID of the images that we want to compute
478 const Rx::LFR::EImage idRGB = Rx::LFR::EImage::TotalFocus_View_Virtual, idDepth = Rx::LFR::EImage::Depth3D_ViewVirtualUndistorted_To_Reference;
479 // ID of he space to compute the total focus
480 const Rx::ELFSpace idSpaceRGB = Rx::ELFSpace::View_VirtualUndistorted;
481 // ID of the space to compute the depth 3D
482 const Rx::ELFSpace idSpaceDepth3D[2] = { idSpaceRGB, Rx::ELFSpace::Reference };
483 /*-------------------------------------------*/
484
485 /*------- Print fps(variables + functions)*/
486 // A string that will be used to display message as the FPS
487 char* printMsg = NULL;
488 // An uint to know where we need to add a new message to the string
489 unsigned int whereToAddMsg;
490 // A fixed size of the string
491 const int SIZE_MSG = 1000;
492 // Time when the first loop began
493 std::chrono::steady_clock::time_point beginAvgStreaming;
494 // Time when the stream has done one loop
495 std::chrono::steady_clock::time_point endStreaming;
502 void addMsgToPrint(const char* title, long long int averageTime, float frameNumber);
506 void resetPrintMsg();
510 void printFPS();
511 /*----------------------------------------*/
512};
HvtIntrinsicsPerspective getRxIntrinsicParams(size_t idxCam)
Get the intrinsic parameters of the camera of ID "idxCam". The intrinsic parameters are found through...
~RaytrixStreamer()
Free all the memories that were allocated during the streaming process.
HvtExtrinsics getRxExtrinsicParams(size_t idxCam)
Get the extrinsic parameters of the camera of ID "idxCam". The extrinsic parameters are found through...
float getMinDepth()
Get the minimum depth.
float getMaxDepth()
Get the maximum depth.
Rx::LFR::CImageQueue & getCamBuffer(size_t idxCam)
Get the camera buffer in the list of camera buffers.
RaytrixStreamer(const uint8_t uuid[VK_UUID_SIZE])
Launch all init functions defined in RaytrixStreamer class.
Parameters for query of the current frames infos.
Intrinsics parameters of a perspective projection.
Description of an RGBD stream.
Export info for images of a stream.
Union of possible intrinsics types data.