push to sdk 4.0.4 (#25)

* add new spatial mapping param

* remove body 70 and simplify automatic offset

* improve auto offset

* clean automatic offset implementation

* hide skeleton by default in body tracking scenes

* add skeleton smoothing

* add remy and update bt scenes

* remove body 70 in anim node

* add new positional tracking mode

* readd height scale

* readd body 70

* ignore metahumans dir

* update wrapper

* disable collisions with world by default

* make zedmanny invisible on bp_remy

---------

Co-authored-by: Jean-Loup Macarit <jeanloup.macarit@stereolabs.com>

.gitignore

@@ -4,6 +4,9 @@
 RuntimeMeshComponent/Binaries/*
 RuntimeMeshComponent/Intermediate/*
 
+# MetaHumans
+Content/MetaHumans/*
+
 # Local tests
 Content/LocalTest/*
 Content/LocalTests/*

Config/DefaultEngine.ini

@@ -2,7 +2,7 @@
 GlobalDefaultGameMode=/Stereolabs/ZED/Blueprints/GameMode/BP_ZED_GameMode.BP_ZED_GameMode_C
 GameInstanceClass=/Script/ZED.ZEDGameInstance
 EditorStartupMap=/Game/ZED/Levels/L_BodyTrackingSingle.L_BodyTrackingSingle
-GameDefaultMap=/Game/ZED/Levels/L_BodyTrackingMulti.L_BodyTrackingMulti
+GameDefaultMap=/Game/Local/L_BodyTrackingSiggraph.L_BodyTrackingSiggraph
 
 
 [/Script/Engine.Engine]

Plugins/Stereolabs/Source/Stereolabs/Private/Core/StereolabsBaseTypes.cpp

@@ -10,7 +10,8 @@ FSlSpatialMappingParameters::FSlSpatialMappingParameters()
 	PresetResolution(ESlSpatialMappingResolution::SMR_Medium),
 	PresetRange(ESlSpatialMappingRange::SMR_Auto),
 	bSaveTexture(false)/*,
-	bUseChunkOnly(bUseChunkOnly)*/
+	bUseChunkOnly(bUseChunkOnly)*/,
+	StabilityCounter(0)
 {
 	MaxRange = sl::SpatialMappingParameters::get(sl::unreal::ToSlType(PresetRange));
 	Resolution = sl::SpatialMappingParameters::get(sl::unreal::ToSlType(PresetResolution));

Plugins/Stereolabs/Source/Stereolabs/Private/Core/StereolabsCameraProxy.cpp

@@ -440,6 +440,7 @@ void USlCameraProxy::Internal_EnableTracking(const FSlPositionalTrackingParamete
 	sl_positional_tracking_parameters.set_as_static = NewTrackingParameters.bSetAsStatic;
 	sl_positional_tracking_parameters.set_floor_as_origin = NewTrackingParameters.bSetFloorAsOrigin;
 	sl_positional_tracking_parameters.set_gravity_as_origin = NewTrackingParameters.bSetGravityAsOrigin;
+	sl_positional_tracking_parameters.mode = (SL_POSITIONAL_TRACKING_MODE)NewTrackingParameters.Mode;
 
 	SL_SCOPE_LOCK(Lock, GrabSection)
 		ErrorCode = (SL_ERROR_CODE)sl_enable_positional_tracking(CameraID, &sl_positional_tracking_parameters, TCHAR_TO_UTF8(*NewTrackingParameters.AreaFilePath));
@@ -909,7 +910,9 @@ bool USlCameraProxy::RetrieveMeshAsync(USlMesh* Mesh)
 	Mesh->Vertices.SetNum(Mesh->NbVertices * 3);
 	Mesh->Triangles.SetNum(Mesh->NbTriangles * 3);
 	Mesh->UVs.SetNum(Mesh->NbVertices * 2);
-	ErrorCode = (SL_ERROR_CODE)sl_retrieve_whole_mesh(CameraID, Mesh->Vertices.GetData(), Mesh->Triangles.GetData(), Mesh->UVs.GetData(), Mesh->Texture.GetData());
+	Mesh->Colors.SetNum(Mesh->NbVertices * 3);
+
+	ErrorCode = (SL_ERROR_CODE)sl_retrieve_whole_mesh(CameraID, Mesh->Vertices.GetData(), Mesh->Triangles.GetData(), Mesh->Colors.GetData(), Mesh->UVs.GetData(), Mesh->Texture.GetData());
 
 	if (ErrorCode != SL_ERROR_CODE_SUCCESS)
 	{

Plugins/Stereolabs/Source/Stereolabs/Private/Core/StereolabsMesh.cpp

@@ -49,8 +49,9 @@ bool USlMesh::Filter(const FSlMeshFilterParameters& MeshFilterParameters/* = FSl
 	Vertices.SetNum(NbVertices * 3);
 	UVs.SetNum(NbVertices * 2);
 	Triangles.SetNum(NbTriangles * 3);
+	Colors.SetNum(NbVertices * 3);
 
-	(SL_ERROR_CODE)sl_retrieve_whole_mesh(GSlCameraProxy->GetCameraID(), Vertices.GetData(), Triangles.GetData(), UVs.GetData(), Texture.GetData());
+	(SL_ERROR_CODE)sl_retrieve_whole_mesh(GSlCameraProxy->GetCameraID(), Vertices.GetData(), Triangles.GetData(), Colors.GetData(), UVs.GetData(), Texture.GetData());
 
 	return bFiltered;
 }
@@ -68,8 +69,10 @@ bool USlMesh::ApplyTexture(bool bSRGB/* = false*/)
 		Vertices.SetNum(NbVertices * 3);
 		UVs.SetNum(NbVertices * 2);
 		Triangles.SetNum(NbTriangles * 3);
+		Colors.SetNum(NbVertices * 3);
+
 		Texture.SetNum(TextureSize[0] * TextureSize[1] * 4);
-		SL_ERROR_CODE err = (SL_ERROR_CODE)sl_retrieve_whole_mesh(GSlCameraProxy->GetCameraID(), Vertices.GetData(), Triangles.GetData(), UVs.GetData(), Texture.GetData());
+		SL_ERROR_CODE err = (SL_ERROR_CODE)sl_retrieve_whole_mesh(GSlCameraProxy->GetCameraID(), Vertices.GetData(), Triangles.GetData(), Colors.GetData(), UVs.GetData(), Texture.GetData());
 
 	}
 
@@ -139,8 +142,9 @@ bool USlMesh::Load(const FString& FilePath, bool bUpdateChunksOnly/* = false*/)
 	Vertices.SetNum(NbVertices * 3);
 	UVs.SetNum(NbVertices * 2);
 	Triangles.SetNum(NbTriangles * 3);
+	Colors.SetNum(NbVertices * 3);
 
-	(SL_ERROR_CODE)sl_retrieve_whole_mesh(GSlCameraProxy->GetCameraID(), Vertices.GetData(), Triangles.GetData(), UVs.GetData(), Texture.GetData());
+	(SL_ERROR_CODE)sl_retrieve_whole_mesh(GSlCameraProxy->GetCameraID(), Vertices.GetData(), Triangles.GetData(), Colors.GetData(), UVs.GetData(), Texture.GetData());
 
 	return bIsLoaded;
 }

Plugins/Stereolabs/Source/Stereolabs/Public/Core/StereolabsBaseTypes.h

@@ -141,13 +141,16 @@ ENUM_CLASS_FLAGS(ESlMemoryType)
  * SDK Video resolutions
  * see sl::RESOLUTION
  */
-UENUM(BlueprintType, Category = "Stereolabs|Enum")
-enum class ESlResolution : uint8
+	UENUM(BlueprintType, Category = "Stereolabs|Enum")
+	enum class ESlResolution : uint8
 {
 	R_HD2K			   		 UMETA(DisplayName = "HD 2K"),
 	R_HD1080		   		 UMETA(DisplayName = "HD 1080p"),
+	R_HD1200			     UMETA(DisplayName = "HD 1200p (ZED X only)"),
 	R_HD720		   			 UMETA(DisplayName = "HD 720p"),
-	R_VGA			   		 UMETA(DisplayName = "VGA")
+	R_SVGA			   		 UMETA(DisplayName = "SVGA (ZED X only)"),
+	R_VGA			   		 UMETA(DisplayName = "VGA"),
+	R_AUTO				     UMETA(DisplayName = "AUTO, 1200p for ZEDX and 720 otherwise")
 };
 
 /*
@@ -498,6 +501,16 @@ enum class ESlSelfCalibrationState : uint8
 	SCS_Success				UMETA(DisplayName = "Success"),
 };
 
+/*
+* Lists the mode of positional tracking that can be used.
+*/
+UENUM(BlueprintType, Category = "Stereolabs|Enum")
+enum class ESlPositionalTrackingMode : uint8
+{
+	PTM_Standard		UMETA(DisplayName = "Standard"),
+	PTM_Quality			UMETA(DisplayName = "Quality")
+};
+
 /*
 * Lists available object tracking state.
 */
@@ -1316,6 +1329,14 @@ struct STEREOLABS_API FSlSpatialMappingParameters
 	UPROPERTY(EditAnywhere, BlueprintReadWrite)
 	ESlSpatialMappingRange PresetRange;
 
+	/**
+	   \brief Control the integration rate of the current depth into the mapping process.
+	   This parameter controls how many times a stable 3D points should be seen before it is integrated into the spatial mapping.
+	   Default value is 0, this will define the stability counter based on the mesh resolution, the higher the resolution, the higher the stability counter.
+	*/
+	UPROPERTY(EditAnywhere, BlueprintReadWrite)
+	int StabilityCounter;
+
 	/*
 	 * Set to true if you want be able to apply texture to your mesh after its creation.
 	 * This option will take more memory.
@@ -1947,7 +1968,8 @@ struct STEREOLABS_API FSlPositionalTrackingParameters
 		bEnableImuFusion(true),
 		bSetAsStatic(false),
 		DepthMinRange(-1),
-		bSetGravityAsOrigin(true)
+		bSetGravityAsOrigin(true),
+		Mode(ESlPositionalTrackingMode::PTM_Standard)
 	{
 	}
 
@@ -2108,6 +2130,13 @@ struct STEREOLABS_API FSlPositionalTrackingParameters
 	*/
 	UPROPERTY(EditAnywhere, BlueprintReadWrite)
 	bool bSetGravityAsOrigin;
+
+	/**
+	* @brief Positional tracking mode used. Can be used to improve accuracy in some type of scene at the cost of longer runtime
+	* default : POSITIONAL_TRACKING_MODE::STANDARD
+	*/
+	UPROPERTY(EditAnywhere, BlueprintReadWrite)
+	ESlPositionalTrackingMode Mode;
 };
 
 /*
@@ -3012,9 +3041,17 @@ struct STEREOLABS_API FSlBodyTrackingRuntimeParameters
 	UPROPERTY(EditAnywhere, BlueprintReadWrite)
 	int MinimumKeypointsThreshold;
 
-	FSlBodyTrackingRuntimeParameters():
+	/**
+	 * @brief this value controls the smoothing of the fitted fused skeleton.
+	 * it is ranged from 0 (low smoothing) and 1 (high smoothing)
+	 */
+	UPROPERTY(EditAnywhere, BlueprintReadWrite)
+	float SkeletonSmoothing;
+
+	FSlBodyTrackingRuntimeParameters() :
 		DetectionConfidenceThreshold(20.0f),
-		MinimumKeypointsThreshold(-1)
+		MinimumKeypointsThreshold(-1),
+		SkeletonSmoothing(0.0f)
 	{}
 };
 

Plugins/Stereolabs/Source/Stereolabs/Public/Core/StereolabsCoreGlobals.h

@@ -2206,6 +2206,7 @@ namespace sl
 			struct SL_BodyTrackingRuntimeParameters BTParameters;
 			BTParameters.detection_confidence_threshold = UnrealData.DetectionConfidenceThreshold;
 			BTParameters.minimum_keypoints_threshold = UnrealData.MinimumKeypointsThreshold;
+			BTParameters.skeleton_smoothing = UnrealData.SkeletonSmoothing;
 
 			return BTParameters;
 		}
@@ -2280,6 +2281,7 @@ namespace sl
 			SpatialMappingParameters.resolution_meter = GetResolution(UnrealData.PresetResolution);
 			SpatialMappingParameters.use_chunk_only = UnrealData.GetUseChunksOnly();
 			SpatialMappingParameters.reverse_vertex_order = true;
+			SpatialMappingParameters.stability_counter = UnrealData.StabilityCounter;
 
 			return SpatialMappingParameters;
 		}

Plugins/Stereolabs/Source/Stereolabs/Public/Core/StereolabsMesh.h

@@ -103,6 +103,8 @@ class STEREOLABS_API USlMesh : public UObject
 
 	TArray<int> Triangles;
 
+	TArray<unsigned char> Colors;
+
 	TArray<float> UVs;
 
 	TArray<unsigned char> Texture;

Plugins/Stereolabs/Source/ThirdParty/sl_zed_c/include/sl/c_api/types_c.h

@@ -286,7 +286,7 @@ enum SL_UNIT {
 \image html CoordinateSystem.png
  */
 enum SL_COORDINATE_SYSTEM {
-	SL_COORDINATE_SYSTEM_IMAGE, /**< Standard coordinates system in computer vision. Used in OpenCV : see here : http://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html */
+	SL_COORDINATE_SYSTEM_IMAGE, /**< Standard coordinates system in computer vision. Used in OpenCV : see <a href="http://docs.opencv.org/2.4/modules/calib3d/doc/camera_calibration_and_3d_reconstruction.html">here</a>. */
 	SL_COORDINATE_SYSTEM_LEFT_HANDED_Y_UP, /**< Left-Handed with Y up and Z forward. Used in Unity with DirectX. */
 	SL_COORDINATE_SYSTEM_RIGHT_HANDED_Y_UP, /**< Right-Handed with Y pointing up and Z backward. Used in OpenGL. */
 	SL_COORDINATE_SYSTEM_RIGHT_HANDED_Z_UP, /**< Right-Handed with Z pointing up and Y forward. Used in 3DSMax. */
@@ -481,6 +481,20 @@ enum SL_POSITIONAL_TRACKING_STATE {
 	SL_POSITIONAL_TRACKING_STATE_SEARCHING_FLOOR_PLANE, /**< The camera is searching for the floor plane to locate itself related to it, the REFERENCE_FRAME::WORLD will be set afterward.*/
 };
 
+/**
+ \enum POSITIONAL_TRACKING_MODE
+ \ingroup PositionalTracking_group
+ \brief Lists the mode of positional tracking that can be used.
+  */
+enum SL_POSITIONAL_TRACKING_MODE {
+	SL_POSITIONAL_TRACKING_MODE_STANDARD, /**< Default mode, best compromise in performance and accuracy */
+	SL_POSITIONAL_TRACKING_MODE_QUALITY, /**< Improve accuracy in more challening scenes such as outdoor repetitive patterns like extensive field. Curently works best with ULTRA depth mode, requires more compute power */
+	///@cond SHOWHIDDEN
+	SL_POSITIONAL_TRACKING_MODE_LAST
+	///@endcond
+};
+
+
 /**
 \brief Lists the different states of spatial memory area export.
  */
@@ -719,7 +733,7 @@ enum SL_BODY_FORMAT
 	 * \brief 70 keypoint model.
 	 * Body model, including feet and full hands models (and simplified face)
 	 */
-	 SL_BODY_FORMAT_BODY_70,
+	SL_BODY_FORMAT_BODY_70,
 };
 
 enum SL_BODY_KEYPOINTS_SELECTION
@@ -1390,6 +1404,12 @@ d	\warning: This mode requires more resources to run, but greatly improves track
 	 * @brief This setting allows you to override 2 of the 3 rotations from initial_world_transform using the IMU gravity
 	 */
 	bool set_gravity_as_origin;
+	/**
+	* @brief Positional tracking mode used. Can be used to improve accuracy in some type of scene at the cost of longer runtime
+	* default : POSITIONAL_TRACKING_MODE::STANDARD
+	*/
+	enum SL_POSITIONAL_TRACKING_MODE mode;
+
 };
 
 /**
@@ -1594,6 +1614,12 @@ struct SL_SpatialMappingParameters {
 	\brief The type of spatial map to be created. This dictates the format that will be used for the mapping(e.g. mesh, point cloud). See \ref SPATIAL_MAP_TYPE
 	 */
 	enum SL_SPATIAL_MAP_TYPE map_type;
+	/**
+		\brief Control the integration rate of the current depth into the mapping process.
+		This parameter controls how many times a stable 3D points should be seen before it is integrated into the spatial mapping.
+		Default value is 0, this will define the stability counter based on the mesh resolution, the higher the resolution, the higher the stability counter.
+	*/
+	int stability_counter;
 };
 
 
@@ -1852,6 +1878,12 @@ struct SL_BodyTrackingRuntimeParameters {
 	 * it is useful for example to remove unstable fitting results when a skeleton is partially occluded.
 	 */
 	int minimum_keypoints_threshold;
+	/**
+	 * @brief this value controls the smoothing of the fitted fused skeleton.
+	 * it is ranged from 0 (low smoothing) and 1 (high smoothing)
+	 * Default is 0;
+	 */
+	float skeleton_smoothing;
 };
 
 /**

Plugins/Stereolabs/Source/ThirdParty/sl_zed_c/include/sl/c_api/zed_interface.h

@@ -657,12 +657,13 @@ extern "C" {
     \param camera_id : id of the camera instance.
     \param vertices : Vertices of the mesh
     \param triangles : Triangles of the mesh.
+    \param colors :  (b,g,r) colors of each vertex.
     \param max_submeshes : Maximum number of submesh that can be handled.
     \param uvs : uvs of the texture.
     \param texture_ptr : Texture of the mesh (if enabled).
     \return SUCCESS if the mesh is retrieved.
      */
-    INTERFACE_API int sl_retrieve_mesh(int camera_id, float* vertices, int* triangles, float* uvs, unsigned char* texture_ptr, const int max_submeshes);
+    INTERFACE_API int sl_retrieve_mesh(int camera_id, float* vertices, int* triangles, unsigned char* colors, float* uvs, unsigned char* texture_ptr, const int max_submeshes);
     /**
     \brief Updates the internal version of the mesh and returns the sizes of the meshes.
     \param camera_id : id of the camera instance.
@@ -683,11 +684,12 @@ extern "C" {
     \param max_submesh : Maximum number of submesh that can be handled.
     \param vertices : Vertices of the chunk
     \param triangles : Triangles of the chunk.
+    \param colors : b,g,r colors of the chunk
     \param uvs : uvs of the texture.
     \param texture_ptr : Texture of the mesh (if enabled).
     \return SUCCESS if the chunk is retrieved.
      */
-    INTERFACE_API int sl_retrieve_chunks(int camera_id, float* vertices, int* triangles, float* uvs, unsigned char* texture_ptr, const int max_submesh);
+    INTERFACE_API int sl_retrieve_chunks(int camera_id, float* vertices, int* triangles, unsigned char* colors, float* uvs, unsigned char* texture_ptr, const int max_submesh);
 
     /**
     \brief Updates the fused point cloud (if spatial map type was FUSED_POINT_CLOUD).
@@ -792,13 +794,14 @@ extern "C" {
     \brief Retrieves the full mesh. Call update_mesh before calling this.
     Vertex and triangles arrays must be at least of the sizes returned by update_mesh (nb_vertices and nbTriangles).
     \param camera_id : id of the camera instance.
-    \param vertices : Vertices of the chunk
-    \param triangles : Triangles of the chunk.
+    \param vertices : Vertices of the mesh
+    \param triangles : Triangles of the mesh.
+    \param colors : (b,g,r) colors of the mesh.
     \param uvs : uvs of the texture.
     \param texture_ptr : Texture of the mesh (if enabled).
     \return SUCCESS if the chunk is retrieved.
      */
-    INTERFACE_API int sl_retrieve_whole_mesh(int camera_id, float* vertices, int* triangles, float* uvs, unsigned char* texture_ptr);
+    INTERFACE_API int sl_retrieve_whole_mesh(int camera_id, float* vertices, int* triangles, unsigned char* colors,  float* uvs, unsigned char* texture_ptr);
     /**
     \brief Loads a saved mesh file.
     \param camera_id : id of the camera instance.