From a33ce2ccbc20914203a3fab66b979ba03379d1f6 Mon Sep 17 00:00:00 2001 From: Hao Guan <10684225+hguandl@users.noreply.github.com> Date: Sun, 8 Mar 2026 10:15:10 +0800 Subject: [PATCH] feat: macOS ScreenCaptureKit (#15939) --- .clang-format | 3 + CMakeLists.txt | 1 + src/MaaCore/CMakeLists.txt | 19 ++ src/MaaCore/Controller/MacSCKHelper.h | 31 ++ src/MaaCore/Controller/MacSCKHelper.mm | 292 ++++++++++++++++++ .../Controller/PlayToolsController.cpp | 88 +++++- src/MaaCore/Controller/PlayToolsController.h | 12 + src/MaaMacGui | 2 +- 8 files changed, 446 insertions(+), 2 deletions(-) create mode 100644 src/MaaCore/Controller/MacSCKHelper.h create mode 100644 src/MaaCore/Controller/MacSCKHelper.mm diff --git a/.clang-format b/.clang-format index 3e410ad160..55dadf0f4d 100644 --- a/.clang-format +++ b/.clang-format @@ -189,3 +189,6 @@ UseTab: Never # VerilogBreakBetweenInstancePorts: # WhitespaceSensitiveMacros: RemoveEmptyLinesInUnwrappedLines: true +--- +Language: ObjC +BasedOnStyle: "WebKit" diff --git a/CMakeLists.txt b/CMakeLists.txt index a928d9a31d..cd9b288e2a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,6 +10,7 @@ option(INSTALL_PYTHON "install python ffi" OFF) option(INSTALL_RESOURCE "install resource" OFF) option(INSTALL_FLATTEN "do not use bin lib include directory" ON) option(WITH_EMULATOR_EXTRAS "build with emulator extras" ${WIN32}) +option(WITH_MAC_SCK "build with macOS ScreenCaptureKit" ${APPLE}) option(WITH_HASH_VERSION "generate version from git hash" OFF) option(BUILD_RESOURCE_UPDATER "build resource updater tool" OFF) diff --git a/src/MaaCore/CMakeLists.txt b/src/MaaCore/CMakeLists.txt index 0b3d630250..3d9546bafc 100644 --- a/src/MaaCore/CMakeLists.txt +++ b/src/MaaCore/CMakeLists.txt @@ -1,5 +1,14 @@ file(GLOB_RECURSE maa_src *.h *.hpp *.cpp) +if(WITH_MAC_SCK) + if(APPLE) + list(APPEND maa_src Controller/MacSCKHelper.mm) + else() + message(WARNING "MacSCK is only supported on Apple platforms, ignoring") + set(WITH_MAC_SCK OFF) + endif() +endif() + add_library(MaaCore SHARED ${maa_src}) if(WITH_EMULATOR_EXTRAS) @@ -12,6 +21,7 @@ if(WITH_EMULATOR_EXTRAS) endif() target_compile_definitions(MaaCore PRIVATE ASST_WITH_EMULATOR_EXTRAS=$) +target_compile_definitions(MaaCore PRIVATE ASST_WITH_MAC_SCK=$) target_include_directories(MaaCore PUBLIC ${PROJECT_SOURCE_DIR}/include PRIVATE .) add_dependencies(MaaCore MaaUtils) @@ -25,6 +35,15 @@ if(LINUX) target_link_libraries(MaaCore pthread dl) endif() +if(APPLE AND WITH_MAC_SCK) + target_link_libraries(MaaCore "-framework Accelerate" + "-framework Foundation" + "-framework CoreGraphics" + "-framework CoreMedia" + "-framework CoreVideo" + "-framework ScreenCaptureKit") +endif() + file(GLOB_RECURSE MaaCore_PUBLIC_HEADERS ${PROJECT_SOURCE_DIR}/include/*.h) target_sources(MaaCore PUBLIC ${MaaCore_PUBLIC_HEADERS}) set_target_properties(MaaCore PROPERTIES PUBLIC_HEADER "${MaaCore_PUBLIC_HEADERS}") diff --git a/src/MaaCore/Controller/MacSCKHelper.h b/src/MaaCore/Controller/MacSCKHelper.h new file mode 100644 index 0000000000..e51921d050 --- /dev/null +++ b/src/MaaCore/Controller/MacSCKHelper.h @@ -0,0 +1,31 @@ +#pragma once + +#ifdef ASST_WITH_MAC_SCK + +#include +#include +#include +#include +#include + +namespace asst +{ +class MacSCKHelper +{ +public: + MacSCKHelper(); + ~MacSCKHelper(); + + bool init(std::string_view bundle_id, std::string_view port, std::pair size, std::array rect); + bool capture(std::vector& bgrData) const; + + static bool hasPermission(); + static bool requestPermission(); + +private: + struct Impl; + std::unique_ptr pImpl; +}; +} // namespace asst + +#endif // ASST_WITH_MAC_SCK diff --git a/src/MaaCore/Controller/MacSCKHelper.mm b/src/MaaCore/Controller/MacSCKHelper.mm new file mode 100644 index 0000000000..ebe160ab7a --- /dev/null +++ b/src/MaaCore/Controller/MacSCKHelper.mm @@ -0,0 +1,292 @@ +#include "MacSCKHelper.h" + +#ifdef ASST_WITH_MAC_SCK + +#import +#import +#import + +#include "Utils/Logger.hpp" + +@interface MacSCKOutput : NSObject { +} + +@property (nonatomic, assign) CVImageBufferRef buffer; + +@property (atomic, assign) BOOL running; + +@end + +@implementation MacSCKOutput + +- (void)stream:(SCStream*)stream didStopWithError:(NSError*)error +{ + if (error) { + Log.error(__FUNCTION__, "| Stream stopped with error:", error.localizedDescription.UTF8String); + } else { + Log.trace(__FUNCTION__, "| Stream stopped without error"); + } + self.running = NO; +} + +- (void)stream:(SCStream*)stream didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer ofType:(SCStreamOutputType)type +{ + if (type != SCStreamOutputTypeScreen) { + return; + } + + const auto newBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + + if (newBuffer && _buffer != newBuffer) { + CFRetain(newBuffer); + if (_buffer) { + CFRelease(_buffer); + } + _buffer = newBuffer; + } +} + +- (void)dealloc +{ + if (_buffer) { + CFRelease(_buffer); + _buffer = NULL; + } + [super dealloc]; +} + +@end + +bool asst::MacSCKHelper::hasPermission() +{ + return CGPreflightScreenCaptureAccess(); +} + +bool asst::MacSCKHelper::requestPermission() +{ + return CGRequestScreenCaptureAccess(); +} + +struct asst::MacSCKHelper::Impl { + ~Impl(); + + SCStream* m_stream = nil; + MacSCKOutput* m_output = nil; + + dispatch_queue_t m_queue = nil; + + bool init(std::string_view bundle_id, std::string_view port, std::pair size, std::array rect); + bool capture(std::vector& bgrData) const; +}; + +asst::MacSCKHelper::Impl::~Impl() +{ + if (m_stream) { + [m_stream stopCaptureWithCompletionHandler:nil]; + [m_stream release]; + m_stream = nil; + } + + [m_output release]; + m_output = nil; + + if (m_queue) { + dispatch_release(m_queue); + m_queue = nil; + } +} + +bool asst::MacSCKHelper::Impl::init(std::string_view bundle_id, std::string_view port, std::pair size, std::array rect) +{ + auto sem = dispatch_semaphore_create(0); + __block bool result = false; + + const auto handler = ^(SCShareableContent* _Nullable content, NSError* _Nullable error) { + if (error) { + Log.error("Cannot get shareable content:", error.localizedDescription.UTF8String); + dispatch_semaphore_signal(sem); + return; + } + + SCWindow* targetWindow = nil; + for (SCWindow* window in content.windows) { + auto bundleID = window.owningApplication.bundleIdentifier.UTF8String; + if (bundleID && bundleID != bundle_id) { + continue; + } + auto title = window.title.UTF8String; + if (title && std::string_view(title).find(port) != std::string_view::npos) { + targetWindow = window; + break; + } + } + if (!targetWindow) { + Log.error("No window found with bundle ID:", bundle_id, ", port:", port); + dispatch_semaphore_signal(sem); + return; + } + + const auto filter = [[SCContentFilter alloc] initWithDesktopIndependentWindow:targetWindow]; + + auto config = [[SCStreamConfiguration alloc] init]; + config.width = size.first; + config.height = size.second; + config.pixelFormat = kCVPixelFormatType_32BGRA; + config.colorSpaceName = kCGColorSpaceSRGB; + config.showsCursor = NO; + + const auto window_height = rect[3]; + const auto content_width = rect[6]; + const auto content_height = rect[7]; + + const auto titlebar_height = window_height - content_height; + if (titlebar_height > 0) { + Log.trace("Titlebar logical height:", titlebar_height); + config.sourceRect = CGRectMake(0, titlebar_height, content_width, content_height); + } + + m_output = [[MacSCKOutput alloc] init]; + m_stream = [[SCStream alloc] initWithFilter:filter configuration:config delegate:m_output]; + + [config release]; + [filter release]; + + m_queue = dispatch_queue_create("plus.maa.MacSCKOutput", NULL); + [m_stream addStreamOutput:m_output + type:SCStreamOutputTypeScreen + sampleHandlerQueue:m_queue + error:&error]; + + if (error) { + Log.error("Cannot add stream output:", error.localizedDescription.UTF8String); + dispatch_semaphore_signal(sem); + return; + } + + [m_stream startCaptureWithCompletionHandler:^(NSError* _Nullable error) { + if (error) { + Log.error("Cannot start capture:", error.localizedDescription.UTF8String); + dispatch_semaphore_signal(sem); + return; + } + + Log.trace("Started capture for window:", targetWindow.title.UTF8String); + m_output.running = YES; + result = true; + dispatch_semaphore_signal(sem); + }]; + }; + + [SCShareableContent getShareableContentExcludingDesktopWindows:YES + onScreenWindowsOnly:YES + completionHandler:handler]; + + dispatch_time_t timeout = dispatch_time(DISPATCH_TIME_NOW, 3 * NSEC_PER_SEC); + if (dispatch_semaphore_wait(sem, timeout) != 0) { + Log.error("Screenshot init timed out."); + result = false; + } + + dispatch_release(sem); + return result; +} + +bool asst::MacSCKHelper::Impl::capture(std::vector& bgrData) const +{ + auto sem = dispatch_semaphore_create(0); + __block bool result = false; + + if (!m_queue || !m_output) { + Log.error("Stream output is not initialized"); + dispatch_release(sem); + return false; + } + + dispatch_async(m_queue, ^{ + if (!m_output.running) { + Log.error("Stream is not running"); + dispatch_semaphore_signal(sem); + return; + } + + auto buffer = m_output.buffer; + if (!buffer) { + Log.error("No image buffer available"); + dispatch_semaphore_signal(sem); + return; + } + + long ret = CVPixelBufferLockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly); + if (ret != kCVReturnSuccess) [[unlikely]] { + Log.error("Failed to lock pixel buffer:", ret); + dispatch_semaphore_signal(sem); + return; + } + + const auto width = CVPixelBufferGetWidth(buffer); + const auto height = CVPixelBufferGetHeight(buffer); + + const auto dstRowBytes = width * 3; + bgrData.resize(height * dstRowBytes); + + vImage_Buffer srcBuffer; + srcBuffer.data = CVPixelBufferGetBaseAddress(buffer); + srcBuffer.height = height; + srcBuffer.width = width; + srcBuffer.rowBytes = CVPixelBufferGetBytesPerRow(buffer); + + vImage_Buffer dstBuffer; + dstBuffer.data = bgrData.data(); + dstBuffer.height = height; + dstBuffer.width = width; + dstBuffer.rowBytes = dstRowBytes; + + ret = vImageConvert_RGBA8888toRGB888(&srcBuffer, &dstBuffer, kvImageNoFlags); + if (ret != kvImageNoError) [[unlikely]] { + Log.error("Failed to convert buffer channels:", ret); + CVPixelBufferUnlockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly); + dispatch_semaphore_signal(sem); + return; + } + + ret = CVPixelBufferUnlockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly); + if (ret != kCVReturnSuccess) [[unlikely]] { + Log.error("Failed to unlock pixel buffer:", ret); + dispatch_semaphore_signal(sem); + return; + } + + result = true; + dispatch_semaphore_signal(sem); + }); + + dispatch_time_t timeout = dispatch_time(DISPATCH_TIME_NOW, 3 * NSEC_PER_SEC); + if (dispatch_semaphore_wait(sem, timeout) != 0) { + Log.error("Screenshot operation timed out."); + result = false; + } + + dispatch_release(sem); + return result; +} + +asst::MacSCKHelper::MacSCKHelper() + : pImpl(std::make_unique()) +{ +} + +asst::MacSCKHelper::~MacSCKHelper() = default; + +bool asst::MacSCKHelper::init(std::string_view bundle_id, std::string_view port, std::pair size, std::array rect) +{ + LogTraceFunction; + return pImpl->init(bundle_id, port, size, rect); +} + +bool asst::MacSCKHelper::capture(std::vector& bgrData) const +{ + LogTraceFunction; + return pImpl->capture(bgrData); +} + +#endif // ASST_WITH_MAC_SCK diff --git a/src/MaaCore/Controller/PlayToolsController.cpp b/src/MaaCore/Controller/PlayToolsController.cpp index fea02bd0a2..1b06725ff3 100644 --- a/src/MaaCore/Controller/PlayToolsController.cpp +++ b/src/MaaCore/Controller/PlayToolsController.cpp @@ -39,6 +39,10 @@ bool asst::PlayToolsController::connect( m_screencap_method = ScreencapMethod::BGR; m_minimal_version = 3; } + else if (config == "MacSCK") { + m_screencap_method = ScreencapMethod::MacSCK; + m_minimal_version = 3; + } return open(); } @@ -72,6 +76,19 @@ bool asst::PlayToolsController::screencap(cv::Mat& image_payload, bool allow_rec return screencap_rgba(image_payload, allow_reconnect); case ScreencapMethod::BGR: return screencap_bgr(image_payload, allow_reconnect); + case ScreencapMethod::MacSCK: { +#if ASST_WITH_MAC_SCK + if (!m_sck_helper.capture(m_screencap_buffer)) { + return false; + } + auto mat = cv::Mat(m_screen_size.second, m_screen_size.first, CV_8UC3, m_screencap_buffer.data()); + mat.copyTo(image_payload); + return true; +#else + Log.error("MacSCK is not built, cannot capture screencap with this method"); + return false; +#endif // ASST_WITH_MAC_SCK + } default: return false; } @@ -333,7 +350,23 @@ bool asst::PlayToolsController::open() return false; } - return check_version() && fetch_screen_res(); + if (!check_version() || !fetch_screen_res()) { + return false; + } + + if (m_screencap_method == ScreencapMethod::MacSCK) { + if (!fetch_frame_rect() || !fetch_bundle_id()) { + return false; + } +#if ASST_WITH_MAC_SCK + return m_sck_helper.init(m_bundle_id, port, m_screen_size, m_frame_rect); +#else + Log.error("MacSCK is not built, fallback to BGR screencap method"); + m_screencap_method = ScreencapMethod::BGR; +#endif // ASST_WITH_MAC_SCK + } + + return true; } bool asst::PlayToolsController::check_version() @@ -410,3 +443,56 @@ bool asst::PlayToolsController::toucher_commit(const TouchPhase phase, const Poi toucher_wait(delay); return true; } + +bool asst::PlayToolsController::fetch_frame_rect() +{ + constexpr char request[] = { 0, 4, 'R', 'E', 'C', 'T' }; + + try { + boost::asio::write(m_socket, boost::asio::buffer(request)); + boost::asio::read(m_socket, boost::asio::buffer(m_frame_rect)); + } + catch (const std::exception& e) { + Log.error("Cannot get frame rectangle:", e.what()); + return false; + } + + for (auto& val : m_frame_rect) { + val = socket_ops::network_to_host_short(val); + } + + return true; +} + +bool asst::PlayToolsController::fetch_bundle_id() +{ + uint32_t length = 0; + + constexpr char request[] = { 0, 4, 'B', 'N', 'D', 'L' }; + + try { + boost::asio::write(m_socket, boost::asio::buffer(request)); + boost::asio::read(m_socket, boost::asio::buffer(&length, sizeof(length))); + length = socket_ops::network_to_host_long(length); + } + catch (const std::exception& e) { + Log.error("Cannot get bundle ID length:", e.what()); + return false; + } + + if (length == 0 || length > BUFSIZ) { + Log.error("Invalid bundle ID length:", length); + return false; + } + + try { + m_bundle_id.resize(length); + boost::asio::read(m_socket, boost::asio::buffer(m_bundle_id.data(), length)); + } + catch (const std::exception& e) { + Log.error("Cannot get bundle ID:", e.what()); + return false; + } + + return true; +} diff --git a/src/MaaCore/Controller/PlayToolsController.h b/src/MaaCore/Controller/PlayToolsController.h index 94bb6cb8f0..e5e508da64 100644 --- a/src/MaaCore/Controller/PlayToolsController.h +++ b/src/MaaCore/Controller/PlayToolsController.h @@ -10,6 +10,7 @@ #include "Common/AsstMsg.h" #include "InstHelper.h" +#include "MacSCKHelper.h" namespace asst { @@ -76,6 +77,7 @@ protected: { RGBA, BGR, + MacSCK, } m_screencap_method = ScreencapMethod::RGBA; enum class TouchPhase @@ -96,6 +98,9 @@ protected: private: unsigned m_minimal_version = 2; + std::string m_bundle_id; + std::array m_frame_rect = { 0, 0, 0, 0, 0, 0, 0, 0 }; + void close(); bool open(); bool check_version(); @@ -104,5 +109,12 @@ private: bool screencap_rgba(cv::Mat& image_payload, bool allow_reconnect); bool screencap_bgr(cv::Mat& image_payload, bool allow_reconnect); + + bool fetch_frame_rect(); + bool fetch_bundle_id(); + +#if ASST_WITH_MAC_SCK + MacSCKHelper m_sck_helper; +#endif // ASST_WITH_MAC_SCK }; } // namespace asst diff --git a/src/MaaMacGui b/src/MaaMacGui index 6b2c6fac32..7eed483002 160000 --- a/src/MaaMacGui +++ b/src/MaaMacGui @@ -1 +1 @@ -Subproject commit 6b2c6fac32cdfac4bea36d4c26df8aae27c80748 +Subproject commit 7eed4830027010efb4f8218b1f29ad51dc77d3d4