Skip to content

Commit

Permalink
Finally working.. using CVPixelBuffer converter provided by mediapipe…
Browse files Browse the repository at this point in the history
… ends up solving the issue also found out that the current graph utputs nothing if there is no face detected, need a new graph.
  • Loading branch information
swittk committed Sep 22, 2021
1 parent 6248873 commit 7f48772
Show file tree
Hide file tree
Showing 18 changed files with 1,438 additions and 131 deletions.
19 changes: 19 additions & 0 deletions FaceMeshIOSFrameworkPod/FaceMeshIOSFrameworkPod.podspec
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
Pod::Spec.new do |s|
s.name = "FaceMeshIOSFrameworkPod"
s.version = "0.1.0"
s.summary = "Downloader for FaceMeshFramework."
s.description = <<-DESC
An extended description of MyFramework project.
DESC
s.homepage = "http://your.homepage/here"
s.license = { :type => 'Copyright', :text => <<-LICENSE
Copyright 2018
Permission is granted to...
LICENSE
}
s.author = { "swittk" => "[email protected]" }
s.source = { :http => 'https://github.com/swittk/MediapipeFaceMeshIOSLibrary/releases/download/2021-09-21/FaceMeshIOSLibFramework.xcframework.zip' }
s.vendored_frameworks = "FaceMeshIOSLibFramework.xcframework"
s.platform = :ios
s.ios.deployment_target = '11.2'
end
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,10 @@

Mediapipe Facemesh fplugin or React Native

WIP
Currently can build, but crashes due to `glog` mutex somehow being stupid and throwing SIGABRT (abort()) when calling inside the react native bridge.
I suspect MediaPipe usage of glog might conflict with React Native's extensive usage.

## Installation

```sh
Expand Down
112 changes: 78 additions & 34 deletions example/ios/MediapipeFacemeshExample.xcodeproj/project.pbxproj
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
archiveVersion = 1;
classes = {
};
objectVersion = 46;
objectVersion = 52;
objects = {

/* Begin PBXBuildFile section */
Expand All @@ -15,6 +15,7 @@
2D02E4BD1E0B4A84006451C7 /* Images.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 13B07FB51A68108700A75B9A /* Images.xcassets */; };
2D02E4BF1E0B4AB3006451C7 /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 13B07FB71A68108700A75B9A /* main.m */; };
2DCD954D1E0B4F2C00145EB5 /* MediapipeFacemeshExampleTests.m in Sources */ = {isa = PBXBuildFile; fileRef = 00E356F21AD99517003FC87E /* MediapipeFacemeshExampleTests.m */; };
31CBCCFC26FA1F6000B713EB /* FaceMeshIOSLibFramework.xcframework in Frameworks */ = {isa = PBXBuildFile; fileRef = 31CBCC6826FA1CCA00B713EB /* FaceMeshIOSLibFramework.xcframework */; };
4C39C56BAD484C67AA576FFA /* libPods-MediapipeFacemeshExample.a in Frameworks */ = {isa = PBXBuildFile; fileRef = CA3E69C5B9553B26FBA2DF04 /* libPods-MediapipeFacemeshExample.a */; };
81AB9BB82411601600AC10FF /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 81AB9BB72411601600AC10FF /* LaunchScreen.storyboard */; };
/* End PBXBuildFile section */
Expand Down Expand Up @@ -49,6 +50,8 @@
13B07FB71A68108700A75B9A /* main.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; name = main.m; path = MediapipeFacemeshExample/main.m; sourceTree = "<group>"; };
2D02E47B1E0B4A5D006451C7 /* MediapipeFacemeshExample-tvOS.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "MediapipeFacemeshExample-tvOS.app"; sourceTree = BUILT_PRODUCTS_DIR; };
2D02E4901E0B4A5D006451C7 /* MediapipeFacemeshExample-tvOSTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = "MediapipeFacemeshExample-tvOSTests.xctest"; sourceTree = BUILT_PRODUCTS_DIR; };
31CBCC6826FA1CCA00B713EB /* FaceMeshIOSLibFramework.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = FaceMeshIOSLibFramework.xcframework; path = ../../Frameworks/FaceMeshIOSLibFramework.xcframework; sourceTree = "<group>"; };
31EE959026F9F6FC00BD24D9 /* FaceMeshIOSLibFramework.xcframework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.xcframework; name = FaceMeshIOSLibFramework.xcframework; path = ../../ios/FaceMeshIOSLibFramework.xcframework; sourceTree = "<group>"; };
47F7ED3B7971BE374F7B8635 /* Pods-MediapipeFacemeshExample.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-MediapipeFacemeshExample.debug.xcconfig"; path = "Target Support Files/Pods-MediapipeFacemeshExample/Pods-MediapipeFacemeshExample.debug.xcconfig"; sourceTree = "<group>"; };
81AB9BB72411601600AC10FF /* LaunchScreen.storyboard */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = file.storyboard; name = LaunchScreen.storyboard; path = MediapipeFacemeshExample/LaunchScreen.storyboard; sourceTree = "<group>"; };
CA3E69C5B9553B26FBA2DF04 /* libPods-MediapipeFacemeshExample.a */ = {isa = PBXFileReference; explicitFileType = archive.ar; includeInIndex = 0; path = "libPods-MediapipeFacemeshExample.a"; sourceTree = BUILT_PRODUCTS_DIR; };
Expand All @@ -69,6 +72,7 @@
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
31CBCCFC26FA1F6000B713EB /* FaceMeshIOSLibFramework.xcframework in Frameworks */,
4C39C56BAD484C67AA576FFA /* libPods-MediapipeFacemeshExample.a in Frameworks */,
);
runOnlyForDeploymentPostprocessing = 0;
Expand Down Expand Up @@ -124,6 +128,8 @@
2D16E6871FA4F8E400B85C8A /* Frameworks */ = {
isa = PBXGroup;
children = (
31CBCC6826FA1CCA00B713EB /* FaceMeshIOSLibFramework.xcframework */,
31EE959026F9F6FC00BD24D9 /* FaceMeshIOSLibFramework.xcframework */,
ED297162215061F000B7C4FE /* JavaScriptCore.framework */,
ED2971642150620600B7C4FE /* JavaScriptCore.framework */,
CA3E69C5B9553B26FBA2DF04 /* libPods-MediapipeFacemeshExample.a */,
Expand All @@ -137,7 +143,6 @@
47F7ED3B7971BE374F7B8635 /* Pods-MediapipeFacemeshExample.debug.xcconfig */,
E00ACF0FDA8BF921659E2F9A /* Pods-MediapipeFacemeshExample.release.xcconfig */,
);
name = Pods;
path = Pods;
sourceTree = "<group>";
};
Expand Down Expand Up @@ -206,7 +211,7 @@
13B07F8E1A680F5B00A75B9A /* Resources */,
00DD1BFF1BD5951E006B06BC /* Bundle React Native code and images */,
C1D60D28B925C94BD88E79D7 /* [CP] Copy Pods Resources */,
3DAEE031DD9A3879521146F9 /* [CP] Embed Pods Frameworks */,
CE69332E72EC5CF7280CDF4D /* [CP] Embed Pods Frameworks */,
);
buildRules = (
);
Expand Down Expand Up @@ -267,6 +272,7 @@
TestTargetID = 13B07F861A680F5B00A75B9A;
};
13B07F861A680F5B00A75B9A = {
DevelopmentTeam = WX9732WYH7;
LastSwiftMigration = 1120;
};
2D02E47A1E0B4A5D006451C7 = {
Expand Down Expand Up @@ -348,7 +354,7 @@
);
runOnlyForDeploymentPostprocessing = 0;
shellPath = /bin/sh;
shellScript = "export NODE_BINARY=node\n../node_modules/react-native/scripts/react-native-xcode.sh";
shellScript = "export NODE_BINARY=node\n../node_modules/react-native/scripts/react-native-xcode.sh\n";
};
2D02E4CB1E0B4B27006451C7 /* Bundle React Native Code And Images */ = {
isa = PBXShellScriptBuildPhase;
Expand All @@ -364,24 +370,6 @@
shellPath = /bin/sh;
shellScript = "export NODE_BINARY=node\n../node_modules/react-native/scripts/react-native-xcode.sh";
};
3DAEE031DD9A3879521146F9 /* [CP] Embed Pods Frameworks */ = {
isa = PBXShellScriptBuildPhase;
buildActionMask = 2147483647;
files = (
);
inputPaths = (
"${PODS_ROOT}/Target Support Files/Pods-MediapipeFacemeshExample/Pods-MediapipeFacemeshExample-frameworks.sh",
"${PODS_XCFRAMEWORKS_BUILD_DIR}/FaceMeshIOSLibFramework/FaceMeshIOSLibFramework.framework/FaceMeshIOSLibFramework",
);
name = "[CP] Embed Pods Frameworks";
outputPaths = (
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/FaceMeshIOSLibFramework.framework",
);
runOnlyForDeploymentPostprocessing = 0;
shellPath = /bin/sh;
shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-MediapipeFacemeshExample/Pods-MediapipeFacemeshExample-frameworks.sh\"\n";
showEnvVarsInLog = 0;
};
4F0A6FC082772762E3E4C96C /* [CP] Check Pods Manifest.lock */ = {
isa = PBXShellScriptBuildPhase;
buildActionMask = 2147483647;
Expand Down Expand Up @@ -422,6 +410,24 @@
shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-MediapipeFacemeshExample/Pods-MediapipeFacemeshExample-resources.sh\"\n";
showEnvVarsInLog = 0;
};
CE69332E72EC5CF7280CDF4D /* [CP] Embed Pods Frameworks */ = {
isa = PBXShellScriptBuildPhase;
buildActionMask = 2147483647;
files = (
);
inputPaths = (
"${PODS_ROOT}/Target Support Files/Pods-MediapipeFacemeshExample/Pods-MediapipeFacemeshExample-frameworks.sh",
"${PODS_XCFRAMEWORKS_BUILD_DIR}/FaceMeshIOSLibFramework/FaceMeshIOSLibFramework.framework/FaceMeshIOSLibFramework",
);
name = "[CP] Embed Pods Frameworks";
outputPaths = (
"${TARGET_BUILD_DIR}/${FRAMEWORKS_FOLDER_PATH}/FaceMeshIOSLibFramework.framework",
);
runOnlyForDeploymentPostprocessing = 0;
shellPath = /bin/sh;
shellScript = "\"${PODS_ROOT}/Target Support Files/Pods-MediapipeFacemeshExample/Pods-MediapipeFacemeshExample-frameworks.sh\"\n";
showEnvVarsInLog = 0;
};
FD10A7F022414F080027D42C /* Start Packager */ = {
isa = PBXShellScriptBuildPhase;
buildActionMask = 2147483647;
Expand Down Expand Up @@ -523,7 +529,11 @@
);
INFOPLIST_FILE = MediapipeFacemeshExampleTests/Info.plist;
IPHONEOS_DEPLOYMENT_TARGET = 10.0;
LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks";
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
"@loader_path/Frameworks",
);
OTHER_LDFLAGS = (
"-ObjC",
"-lc++",
Expand All @@ -542,7 +552,11 @@
COPY_PHASE_STRIP = NO;
INFOPLIST_FILE = MediapipeFacemeshExampleTests/Info.plist;
IPHONEOS_DEPLOYMENT_TARGET = 10.0;
LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks";
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
"@loader_path/Frameworks",
);
OTHER_LDFLAGS = (
"-ObjC",
"-lc++",
Expand All @@ -561,15 +575,20 @@
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_ENABLE_MODULES = YES;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = WX9732WYH7;
ENABLE_BITCODE = NO;
INFOPLIST_FILE = MediapipeFacemeshExample/Info.plist;
LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks";
IPHONEOS_DEPLOYMENT_TARGET = 11.2;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
);
OTHER_LDFLAGS = (
"$(inherited)",
"-ObjC",
"-lc++",
);
PRODUCT_BUNDLE_IDENTIFIER = com.example.reactnativemediapipefacemesh;
PRODUCT_BUNDLE_IDENTIFIER = com.swittssoftware.reactnativemediapipefacemesh;
PRODUCT_NAME = MediapipeFacemeshExample;
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
SWIFT_VERSION = 5.0;
Expand All @@ -584,14 +603,19 @@
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
CLANG_ENABLE_MODULES = YES;
CURRENT_PROJECT_VERSION = 1;
DEVELOPMENT_TEAM = WX9732WYH7;
INFOPLIST_FILE = MediapipeFacemeshExample/Info.plist;
LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks";
IPHONEOS_DEPLOYMENT_TARGET = 11.2;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
);
OTHER_LDFLAGS = (
"$(inherited)",
"-ObjC",
"-lc++",
);
PRODUCT_BUNDLE_IDENTIFIER = com.example.reactnativemediapipefacemesh;
PRODUCT_BUNDLE_IDENTIFIER = com.swittssoftware.reactnativemediapipefacemesh;
PRODUCT_NAME = MediapipeFacemeshExample;
SWIFT_VERSION = 5.0;
VERSIONING_SYSTEM = "apple-generic";
Expand All @@ -611,7 +635,10 @@
ENABLE_TESTABILITY = YES;
GCC_NO_COMMON_BLOCKS = YES;
INFOPLIST_FILE = "MediapipeFacemeshExample-tvOS/Info.plist";
LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks";
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
);
OTHER_LDFLAGS = (
"$(inherited)",
"-ObjC",
Expand All @@ -638,7 +665,10 @@
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
GCC_NO_COMMON_BLOCKS = YES;
INFOPLIST_FILE = "MediapipeFacemeshExample-tvOS/Info.plist";
LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks";
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
);
OTHER_LDFLAGS = (
"$(inherited)",
"-ObjC",
Expand All @@ -664,7 +694,11 @@
ENABLE_TESTABILITY = YES;
GCC_NO_COMMON_BLOCKS = YES;
INFOPLIST_FILE = "MediapipeFacemeshExample-tvOSTests/Info.plist";
LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks";
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
"@loader_path/Frameworks",
);
OTHER_LDFLAGS = (
"$(inherited)",
"-ObjC",
Expand All @@ -690,7 +724,11 @@
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
GCC_NO_COMMON_BLOCKS = YES;
INFOPLIST_FILE = "MediapipeFacemeshExample-tvOSTests/Info.plist";
LD_RUNPATH_SEARCH_PATHS = "$(inherited) @executable_path/Frameworks @loader_path/Frameworks";
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
"@loader_path/Frameworks",
);
OTHER_LDFLAGS = (
"$(inherited)",
"-ObjC",
Expand Down Expand Up @@ -752,7 +790,10 @@
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 10.0;
LD_RUNPATH_SEARCH_PATHS = "/usr/lib/swift $(inherited)";
LD_RUNPATH_SEARCH_PATHS = (
/usr/lib/swift,
"$(inherited)",
);
LIBRARY_SEARCH_PATHS = (
"\"$(TOOLCHAIN_DIR)/usr/lib/swift/$(PLATFORM_NAME)\"",
"\"$(TOOLCHAIN_DIR)/usr/lib/swift-5.0/$(PLATFORM_NAME)\"",
Expand Down Expand Up @@ -805,7 +846,10 @@
GCC_WARN_UNUSED_FUNCTION = YES;
GCC_WARN_UNUSED_VARIABLE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 10.0;
LD_RUNPATH_SEARCH_PATHS = "/usr/lib/swift $(inherited)";
LD_RUNPATH_SEARCH_PATHS = (
/usr/lib/swift,
"$(inherited)",
);
LIBRARY_SEARCH_PATHS = (
"\"$(TOOLCHAIN_DIR)/usr/lib/swift/$(PLATFORM_NAME)\"",
"\"$(TOOLCHAIN_DIR)/usr/lib/swift-5.0/$(PLATFORM_NAME)\"",
Expand Down
4 changes: 2 additions & 2 deletions example/ios/MediapipeFacemeshExample/AppDelegate.h
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@

#import <React/RCTBridgeDelegate.h>
#import <UIKit/UIKit.h>
#import <UMCore/UMAppDelegateWrapper.h>

@interface AppDelegate : UIResponder <UIApplicationDelegate, RCTBridgeDelegate>

@interface AppDelegate : UMAppDelegateWrapper <UIApplicationDelegate, RCTBridgeDelegate>
@property (nonatomic, strong) UIWindow *window;

@end
15 changes: 15 additions & 0 deletions example/ios/MediapipeFacemeshExample/AppDelegate.m
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@
#import <React/RCTBridge.h>
#import <React/RCTBundleURLProvider.h>
#import <React/RCTRootView.h>
#import <UMCore/UMModuleRegistry.h>
#import <UMReactNativeAdapter/UMNativeModulesProxy.h>
#import <UMReactNativeAdapter/UMModuleRegistryAdapter.h>

#ifdef FB_SONARKIT_ENABLED
#import <FlipperKit/FlipperClient.h>
Expand All @@ -29,13 +32,17 @@ static void InitializeFlipper(UIApplication *application) {
}
#endif

@interface AppDelegate () <RCTBridgeDelegate>
@property (nonatomic, strong) UMModuleRegistryAdapter *moduleRegistryAdapter;
@end
@implementation AppDelegate

- (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions
{
#ifdef FB_SONARKIT_ENABLED
InitializeFlipper(application);
#endif
self.moduleRegistryAdapter = [[UMModuleRegistryAdapter alloc] initWithModuleRegistryProvider:[[UMModuleRegistryProvider alloc] init]];
RCTBridge *bridge = [[RCTBridge alloc] initWithDelegate:self launchOptions:launchOptions];
RCTRootView *rootView = [[RCTRootView alloc] initWithBridge:bridge
moduleName:@"MediapipeFacemeshExample"
Expand All @@ -48,9 +55,17 @@ - (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(
rootViewController.view = rootView;
self.window.rootViewController = rootViewController;
[self.window makeKeyAndVisible];
[super application:application didFinishLaunchingWithOptions:launchOptions];
return YES;
}

- (NSArray<id<RCTBridgeModule>> *)extraModulesForBridge:(RCTBridge *)bridge {
NSArray<id<RCTBridgeModule>> *extraModules = [_moduleRegistryAdapter extraModulesForBridge:bridge];
// If you'd like to export some custom RCTBridgeModules that are not Expo modules, add them here!
return extraModules;
}


- (NSURL *)sourceURLForBridge:(RCTBridge *)bridge
{
#if DEBUG
Expand Down
2 changes: 2 additions & 0 deletions example/ios/MediapipeFacemeshExample/Info.plist
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@
</dict>
<key>NSLocationWhenInUseUsageDescription</key>
<string></string>
<key>NSPhotoLibraryUsageDescription</key>
<string>This app accesses the photo library to pick photos to evaluate</string>
<key>UILaunchStoryboardName</key>
<string>LaunchScreen</string>
<key>UIRequiredDeviceCapabilities</key>
Expand Down
12 changes: 7 additions & 5 deletions example/ios/Podfile
Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
require_relative '../node_modules/react-native/scripts/react_native_pods'
require_relative '../node_modules/@react-native-community/cli-platform-ios/native_modules'
require File.join(`node --print "require.resolve('react-native/package.json')"`, "../scripts/react_native_pods")
require File.join(`node --print "require.resolve('@react-native-community/cli-platform-ios/package.json')"`, "../native_modules")
require File.join(`node --print "require.resolve('react-native-unimodules/package.json')"`, "../cocoapods")

platform :ios, '10.0'
platform :ios, '11.0'

target 'MediapipeFacemeshExample' do
use_unimodules!
config = use_native_modules!

use_react_native!(:path => config["reactNativePath"])
Expand All @@ -14,8 +16,8 @@ target 'MediapipeFacemeshExample' do
#
# Note that if you have use_frameworks! enabled, Flipper will not work and
# you should disable these next few lines.
# use_flipper!({ 'Flipper' => '0.80.0' })
# use_flipper!({ 'Flipper' => '0.92.0' }) # Was '0.80.0' which doesn't work
# post_install do |installer|
# flipper_post_install(installer)
# flipper_post_install(installer)
# end
end
Loading

0 comments on commit 7f48772

Please sign in to comment.