converted to cocoapods and now should be working

swittk · Sep 22, 2021 · e00a348 · e00a348
1 parent 7f48772
commit e00a348
Show file tree

Hide file tree

Showing 9 changed files with 130 additions and 85 deletions.
diff --git a/FaceMeshIOSFrameworkPod/FaceMeshIOSFrameworkPod.podspec b/FaceMeshIOSFrameworkPod/FaceMeshIOSFrameworkPod.podspec
diff --git a/README.md b/README.md
@@ -6,6 +6,20 @@ WIP
 Currently can build, but crashes due to `glog` mutex somehow being stupid and throwing SIGABRT (abort()) when calling inside the react native bridge.
 I suspect MediaPipe usage of glog might conflict with React Native's extensive usage.
 
+Changing this zone at /ios/Pods/glog/src/base/mutex.h makes it work (Basically making glog's mutex useless).
+From 
+```
+#define SAFE_PTHREAD(fncall)  do {   /* run fncall if is_safe_ is true */  \
+  if (is_safe_ && fncall(&mutex_) != 0) abort();                           \
+} while (0)
+```
+to
+```
+#define SAFE_PTHREAD(fncall)  do {   /* run fncall if is_safe_ is true */  \
+if (is_safe_ && fncall(&mutex_) != 0) {} /*{ printf("killing app because is safe and fncall is not zero :("); abort();}*/\
+} while (0)
+```
+
 ## Installation
 
 ```sh

diff --git a/example/ios/MediapipeFacemeshExample.xcodeproj/project.pbxproj b/example/ios/MediapipeFacemeshExample.xcodeproj/project.pbxproj
@@ -3,7 +3,7 @@
 	archiveVersion = 1;
 	classes = {
 	};
-	objectVersion = 52;
+	objectVersion = 46;
 	objects = {
 
 /* Begin PBXBuildFile section */
@@ -15,7 +15,6 @@
 		2D02E4BD1E0B4A84006451C7 /* Images.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 13B07FB51A68108700A75B9A /* Images.xcassets */; };
 		2D02E4BF1E0B4AB3006451C7 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 13B07FB71A68108700A75B9A /* main.m */; };
 		2DCD954D1E0B4F2C00145EB5 /* MediapipeFacemeshExampleTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 00E356F21AD99517003FC87E /* MediapipeFacemeshExampleTests.m */; };
-		31CBCCFC26FA1F6000B713EB /* FaceMeshIOSLibFramework.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 31CBCC6826FA1CCA00B713EB /* FaceMeshIOSLibFramework.xcframework */; };
 		4C39C56BAD484C67AA576FFA /* libPods-MediapipeFacemeshExample.a in Frameworks */ = {isa = PBXBuildFile; fileRef = CA3E69C5B9553B26FBA2DF04 /* libPods-MediapipeFacemeshExample.a */; };
 		81AB9BB82411601600AC10FF /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 81AB9BB72411601600AC10FF /* LaunchScreen.storyboard */; };
 /* End PBXBuildFile section */
@@ -72,7 +71,6 @@
 			isa = PBXFrameworksBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
-				31CBCCFC26FA1F6000B713EB /* FaceMeshIOSLibFramework.xcframework in Frameworks */,
 				4C39C56BAD484C67AA576FFA /* libPods-MediapipeFacemeshExample.a in Frameworks */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
@@ -529,11 +527,7 @@
 				);
 				INFOPLIST_FILE = MediapipeFacemeshExampleTests/Info.plist;
 				IPHONEOS_DEPLOYMENT_TARGET = 10.0;
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/Frameworks",
-					"@loader_path/Frameworks",
-				);
+				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks";
 				OTHER_LDFLAGS = (
 					"-ObjC",
 					"-lc++",
@@ -552,11 +546,7 @@
 				COPY_PHASE_STRIP = NO;
 				INFOPLIST_FILE = MediapipeFacemeshExampleTests/Info.plist;
 				IPHONEOS_DEPLOYMENT_TARGET = 10.0;
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/Frameworks",
-					"@loader_path/Frameworks",
-				);
+				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks";
 				OTHER_LDFLAGS = (
 					"-ObjC",
 					"-lc++",
@@ -579,10 +569,7 @@
 				ENABLE_BITCODE = NO;
 				INFOPLIST_FILE = MediapipeFacemeshExample/Info.plist;
 				IPHONEOS_DEPLOYMENT_TARGET = 11.2;
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/Frameworks",
-				);
+				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks";
 				OTHER_LDFLAGS = (
 					"$(inherited)",
 					"-ObjC",
@@ -606,10 +593,7 @@
 				DEVELOPMENT_TEAM = WX9732WYH7;
 				INFOPLIST_FILE = MediapipeFacemeshExample/Info.plist;
 				IPHONEOS_DEPLOYMENT_TARGET = 11.2;
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/Frameworks",
-				);
+				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks";
 				OTHER_LDFLAGS = (
 					"$(inherited)",
 					"-ObjC",
@@ -635,10 +619,7 @@
 				ENABLE_TESTABILITY = YES;
 				GCC_NO_COMMON_BLOCKS = YES;
 				INFOPLIST_FILE = "MediapipeFacemeshExample-tvOS/Info.plist";
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/Frameworks",
-				);
+				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks";
 				OTHER_LDFLAGS = (
 					"$(inherited)",
 					"-ObjC",
@@ -665,10 +646,7 @@
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
 				GCC_NO_COMMON_BLOCKS = YES;
 				INFOPLIST_FILE = "MediapipeFacemeshExample-tvOS/Info.plist";
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/Frameworks",
-				);
+				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks";
 				OTHER_LDFLAGS = (
 					"$(inherited)",
 					"-ObjC",
@@ -694,11 +672,7 @@
 				ENABLE_TESTABILITY = YES;
 				GCC_NO_COMMON_BLOCKS = YES;
 				INFOPLIST_FILE = "MediapipeFacemeshExample-tvOSTests/Info.plist";
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/Frameworks",
-					"@loader_path/Frameworks",
-				);
+				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks";
 				OTHER_LDFLAGS = (
 					"$(inherited)",
 					"-ObjC",
@@ -724,11 +698,7 @@
 				DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
 				GCC_NO_COMMON_BLOCKS = YES;
 				INFOPLIST_FILE = "MediapipeFacemeshExample-tvOSTests/Info.plist";
-				LD_RUNPATH_SEARCH_PATHS = (
-					"$(inherited)",
-					"@executable_path/Frameworks",
-					"@loader_path/Frameworks",
-				);
+				LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks";
 				OTHER_LDFLAGS = (
 					"$(inherited)",
 					"-ObjC",
@@ -790,10 +760,7 @@
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
 				IPHONEOS_DEPLOYMENT_TARGET = 10.0;
-				LD_RUNPATH_SEARCH_PATHS = (
-					/usr/lib/swift,
-					"$(inherited)",
-				);
+				LD_RUNPATH_SEARCH_PATHS = "/usr/lib/swift $(inherited)";
 				LIBRARY_SEARCH_PATHS = (
 					"\"$(TOOLCHAIN_DIR)/usr/lib/swift/$(PLATFORM_NAME)\"",
 					"\"$(TOOLCHAIN_DIR)/usr/lib/swift-5.0/$(PLATFORM_NAME)\"",
@@ -846,10 +813,7 @@
 				GCC_WARN_UNUSED_FUNCTION = YES;
 				GCC_WARN_UNUSED_VARIABLE = YES;
 				IPHONEOS_DEPLOYMENT_TARGET = 10.0;
-				LD_RUNPATH_SEARCH_PATHS = (
-					/usr/lib/swift,
-					"$(inherited)",
-				);
+				LD_RUNPATH_SEARCH_PATHS = "/usr/lib/swift $(inherited)";
 				LIBRARY_SEARCH_PATHS = (
 					"\"$(TOOLCHAIN_DIR)/usr/lib/swift/$(PLATFORM_NAME)\"",
 					"\"$(TOOLCHAIN_DIR)/usr/lib/swift-5.0/$(PLATFORM_NAME)\"",

diff --git a/example/ios/Podfile.lock b/example/ios/Podfile.lock
@@ -23,6 +23,7 @@ PODS:
   - ExpoModulesCore/Interfaces (0.2.0):
     - ExpoModulesCore/Core
     - UMCore
+  - FaceMeshIOSLibFramework (0.5.0)
   - FBLazyVector (0.63.4)
   - FBReactNativeSpec (0.63.4):
     - Folly (= 2020.01.13.00)
@@ -209,6 +210,7 @@ PODS:
     - React-jsi (= 0.63.4)
   - React-jsinspector (0.63.4)
   - react-native-mediapipe-facemesh (0.1.0):
+    - FaceMeshIOSLibFramework
     - PromisesObjC
     - React-Core
   - React-RCTActionSheet (0.63.4):
@@ -324,6 +326,7 @@ DEPENDENCIES:
 SPEC REPOS:
   trunk:
     - boost-for-react-native
+    - FaceMeshIOSLibFramework
     - PromisesObjC
 
 EXTERNAL SOURCES:
@@ -408,6 +411,7 @@ SPEC CHECKSUMS:
   EXImageLoader: d3531a3fe530b22925c19977cb53bb43e3821fe6
   EXImagePicker: 414a3663910fd6604248e84c9c77a0501ae0d18b
   ExpoModulesCore: 2734852616127a6c1fc23012197890a6f3763dc7
+  FaceMeshIOSLibFramework: 3ee7703d7694442df74a4d722b14a415d8da2d3f
   FBLazyVector: 3bb422f41b18121b71783a905c10e58606f7dc3e
   FBReactNativeSpec: f2c97f2529dd79c083355182cc158c9f98f4bd6e
   Folly: b73c3869541e86821df3c387eb0af5f65addfab4
@@ -423,7 +427,7 @@ SPEC CHECKSUMS:
   React-jsi: a0418934cf48f25b485631deb27c64dc40fb4c31
   React-jsiexecutor: 93bd528844ad21dc07aab1c67cb10abae6df6949
   React-jsinspector: 58aef7155bc9a9683f5b60b35eccea8722a4f53a
-  react-native-mediapipe-facemesh: f4990df3fa69998428ec2d0661c00fc10f3bf399
+  react-native-mediapipe-facemesh: fc28ac6ba23a46e60991cdaa4ec9571906433196
   React-RCTActionSheet: 89a0ca9f4a06c1f93c26067af074ccdce0f40336
   React-RCTAnimation: 1bde3ecc0c104c55df246eda516e0deb03c4e49b
   React-RCTBlob: a97d378b527740cc667e03ebfa183a75231ab0f0
@@ -442,4 +446,4 @@ SPEC CHECKSUMS:
 
 PODFILE CHECKSUM: a377a14c0f6f5468056335c55825806eda3bf793
 
-COCOAPODS: 1.10.1
+COCOAPODS: 1.11.2
diff --git a/example/src/FacemeshTester.tsx b/example/src/FacemeshTester.tsx
@@ -2,7 +2,7 @@ import * as React from 'react';
 import { Asset } from 'expo-asset';
 import * as ImagePicker from 'expo-image-picker';
 
-import { StyleSheet, View, Text, Alert, Button, Image } from 'react-native';
+import { StyleSheet, View, Text, Alert, Button, Image, StyleProp, ViewStyle } from 'react-native';
 import MediapipeFacemesh from 'react-native-mediapipe-facemesh';
 
 const FacemeshTester = React.memo((props: {
@@ -19,11 +19,14 @@ const FacemeshTester = React.memo((props: {
         console.log('result nil');
         return;
       }
+      console.log('results shape', getShape(res)); //  [ 1, 1, 468, 3 ]
       setResult(res);
-
-      console.log('results shape', getShape(res));
-      console.log('results are', res);
+      // console.log('results are', res);
     }} />
+    <RenderFaceBox
+      face={result?.[0][0]}
+      imageUri={uri}
+    />
     <Image source={{ uri: uri }} style={{ width: 320, height: 320 }} resizeMode='contain' />
   </View>
 })
@@ -37,9 +40,76 @@ function getShape(obj:any[]): number[]|undefined {
         dims.push(...nextShapes);
       }
     }
+    return dims;
   }
   else {
     return undefined;
   }
 }
-export default FacemeshTester;
+export default FacemeshTester;
+
+
+
+export const RenderFaceBox = React.memo((props: {
+  face?: number[][],
+  style?: StyleProp<ViewStyle>,
+  imageUri?: string
+}) => {
+  const {
+    style,
+    face,
+    imageUri
+  } = props;
+  const [imSize, setImSize] = React.useState({ width: 250, height: 250 });
+  React.useEffect(() => {
+    if (!imageUri) return;
+    (async () => {
+      const prom = new Promise<{ width: number, height: number }>((resolve, reject) => {
+        Image.getSize(imageUri, (w, h) => {
+          resolve({ width: w, height: h });
+        }, reject);
+      });
+      const size = await prom;
+      setImSize(size);
+    })();
+  }, [imageUri]);
+  // shape of FaceMesh output is expected to be (1,1,1,1404)
+  // console.log('rendered', faces.length, 'faces');
+  // const validArray = React.useMemo(() => {
+  //   return arrayEqual(face.shape, [1, 1, 1, 1404]);
+  // }, [face]);
+  const landmarkCircles = React.useMemo(()=>{
+    if(!face) return undefined;
+    const alldata = face;
+    const datalen = alldata.length;
+    const landmarkCircles: JSX.Element[] = [];
+    for (let i = 0; i < datalen; i += 1) {
+      const point = alldata[i];
+      const x = point[0];
+      const y = point[1];
+      // const z = data[i + 2];
+      landmarkCircles.push(<View
+        key={`landmark_${i}`}
+        style={{
+          width: 1.5, height: 1.5, borderRadius: 0.5,
+          left: `${x * 100}%`,
+          top: `${y * 100}%`,
+          backgroundColor: 'blue',
+          position: 'absolute'
+        }}
+      />);
+    }
+    return landmarkCircles;
+  }, [face]);
+
+  return <View
+    style={[{ width: 250, height: 250 }, style]}
+  >
+    <Image source={{ uri: imageUri }} style={{ aspectRatio: imSize.width / imSize.height, flex: 1 }} resizeMode='contain' />
+    <View style={StyleSheet.absoluteFill} pointerEvents='none'>
+      <View style={{ aspectRatio: imSize.width / imSize.height, flex: 1 }}>
+        {landmarkCircles}
+      </View>
+    </View>
+  </View>;
+})
diff --git a/ios/ChunkBasedFaceMeshRunner.m b/ios/ChunkBasedFaceMeshRunner.m
@@ -137,7 +137,7 @@ -(void)processFilesAtPathsQueuer:(NSArray <NSString *>*)filePaths {
         NSString *path = [filePath hasPrefix:@"file://"] ? [filePath substringFromIndex:7] : filePath;
         UIImage *image = [UIImage imageWithContentsOfFile:path];
         NSLog(@"image %@", image);
-        CVPixelBufferRef buffer = CreateCVPixelBufferFromCGImage(image.CGImage);//[self createIOSurfaceBackedPixelBufferFromCGImage:image.CGImage];
+        CVPixelBufferRef buffer = CreateCVPixelBufferFromCGImage([self orientationUpImage:image].CGImage);//[self createIOSurfaceBackedPixelBufferFromCGImage:image.CGImage];
         NSLog(@"buffer %ld", CVPixelBufferGetWidth(buffer));
         [faceMesh processVideoFrame:buffer];
         NSLog(@"sent in frame, the delegate is %@", faceMesh.delegate);
@@ -162,7 +162,7 @@ -(void)processFilesAtPathsQueuer:(NSArray <NSString *>*)filePaths {
 -(void)processImagesQueuer:(NSArray <UIImage *>*)images {
     FaceMeshIOSLib *faceMesh = self.faceMesh;
     for(UIImage *image in images) {
-        CVPixelBufferRef buffer = CreateCVPixelBufferFromCGImage(image.CGImage);
+        CVPixelBufferRef buffer = CreateCVPixelBufferFromCGImage([self orientationUpImage:image].CGImage);
         [faceMesh processVideoFrame:buffer];
         CVPixelBufferRelease(buffer);
     }
@@ -231,4 +231,21 @@ - (CVPixelBufferRef)createIOSurfaceBackedPixelBufferFromCGImage:(CGImageRef)imag
     return pxbuffer;
 }
 
+-(UIImage *)orientationUpImage:(UIImage *)image {
+    if(image.imageOrientation == UIImageOrientationUp) {
+        return image;
+    }
+    CGSize size = image.size;
+    if(image.scale != 1) {
+        size.width *= image.scale;
+        size.height *= image.scale;
+    }
+    UIGraphicsBeginImageContext(size);
+    [image drawInRect:CGRectMake(0,0,size.width,size.height)];
+    UIImage* newImage = UIGraphicsGetImageFromCurrentImageContext();
+    UIGraphicsEndImageContext();
+    return newImage;
+}
+
+
 @end