feat: macOS ScreenCaptureKit (#15939)

.clang-format

@@ -189,3 +189,6 @@ UseTab: Never
 # VerilogBreakBetweenInstancePorts:
 # WhitespaceSensitiveMacros:
 RemoveEmptyLinesInUnwrappedLines: true
+---
+Language: ObjC
+BasedOnStyle: "WebKit"

CMakeLists.txt

@@ -10,6 +10,7 @@ option(INSTALL_PYTHON "install python ffi" OFF)
 option(INSTALL_RESOURCE "install resource" OFF)
 option(INSTALL_FLATTEN "do not use bin lib include directory" ON)
 option(WITH_EMULATOR_EXTRAS "build with emulator extras" ${WIN32})
+option(WITH_MAC_SCK "build with macOS ScreenCaptureKit" ${APPLE})
 option(WITH_HASH_VERSION "generate version from git hash" OFF)
 option(BUILD_RESOURCE_UPDATER "build resource updater tool" OFF)
 

src/MaaCore/CMakeLists.txt

@@ -1,5 +1,14 @@
 file(GLOB_RECURSE maa_src *.h *.hpp *.cpp)
 
+if(WITH_MAC_SCK)
+    if(APPLE)
+        list(APPEND maa_src Controller/MacSCKHelper.mm)
+    else()
+        message(WARNING "MacSCK is only supported on Apple platforms, ignoring")
+        set(WITH_MAC_SCK OFF)
+    endif()
+endif()
+
 add_library(MaaCore SHARED ${maa_src})
 
 if(WITH_EMULATOR_EXTRAS)
@@ -12,6 +21,7 @@ if(WITH_EMULATOR_EXTRAS)
 endif()
 
 target_compile_definitions(MaaCore PRIVATE ASST_WITH_EMULATOR_EXTRAS=$<BOOL:${WITH_EMULATOR_EXTRAS}>)
+target_compile_definitions(MaaCore PRIVATE ASST_WITH_MAC_SCK=$<BOOL:${WITH_MAC_SCK}>)
 
 target_include_directories(MaaCore PUBLIC ${PROJECT_SOURCE_DIR}/include PRIVATE .)
 add_dependencies(MaaCore MaaUtils)
@@ -25,6 +35,15 @@ if(LINUX)
     target_link_libraries(MaaCore pthread dl)
 endif()
 
+if(APPLE AND WITH_MAC_SCK)
+    target_link_libraries(MaaCore "-framework Accelerate"
+                                  "-framework Foundation"
+                                  "-framework CoreGraphics"
+                                  "-framework CoreMedia"
+                                  "-framework CoreVideo"
+                                  "-framework ScreenCaptureKit")
+endif()
+
 file(GLOB_RECURSE MaaCore_PUBLIC_HEADERS ${PROJECT_SOURCE_DIR}/include/*.h)
 target_sources(MaaCore PUBLIC ${MaaCore_PUBLIC_HEADERS})
 set_target_properties(MaaCore PROPERTIES PUBLIC_HEADER "${MaaCore_PUBLIC_HEADERS}")

src/MaaCore/Controller/MacSCKHelper.h

@@ -0,0 +1,31 @@
+#pragma once
+
+#ifdef ASST_WITH_MAC_SCK
+
+#include <array>
+#include <cstdint>
+#include <memory>
+#include <string_view>
+#include <vector>
+
+namespace asst
+{
+class MacSCKHelper
+{
+public:
+    MacSCKHelper();
+    ~MacSCKHelper();
+
+    bool init(std::string_view bundle_id, std::string_view port, std::pair<int, int> size, std::array<int16_t, 8> rect);
+    bool capture(std::vector<uint8_t>& bgrData) const;
+
+    static bool hasPermission();
+    static bool requestPermission();
+
+private:
+    struct Impl;
+    std::unique_ptr<Impl> pImpl;
+};
+} // namespace asst
+
+#endif // ASST_WITH_MAC_SCK

src/MaaCore/Controller/MacSCKHelper.mm

@@ -0,0 +1,292 @@
+#include "MacSCKHelper.h"
+
+#ifdef ASST_WITH_MAC_SCK
+
+#import <Accelerate/Accelerate.h>
+#import <CoreGraphics/CoreGraphics.h>
+#import <ScreenCaptureKit/ScreenCaptureKit.h>
+
+#include "Utils/Logger.hpp"
+
+@interface MacSCKOutput : NSObject <SCStreamDelegate, SCStreamOutput> {
+}
+
+@property (nonatomic, assign) CVImageBufferRef buffer;
+
+@property (atomic, assign) BOOL running;
+
+@end
+
+@implementation MacSCKOutput
+
+- (void)stream:(SCStream*)stream didStopWithError:(NSError*)error
+{
+    if (error) {
+        Log.error(__FUNCTION__, "| Stream stopped with error:", error.localizedDescription.UTF8String);
+    } else {
+        Log.trace(__FUNCTION__, "| Stream stopped without error");
+    }
+    self.running = NO;
+}
+
+- (void)stream:(SCStream*)stream didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer ofType:(SCStreamOutputType)type
+{
+    if (type != SCStreamOutputTypeScreen) {
+        return;
+    }
+
+    const auto newBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
+
+    if (newBuffer && _buffer != newBuffer) {
+        CFRetain(newBuffer);
+        if (_buffer) {
+            CFRelease(_buffer);
+        }
+        _buffer = newBuffer;
+    }
+}
+
+- (void)dealloc
+{
+    if (_buffer) {
+        CFRelease(_buffer);
+        _buffer = NULL;
+    }
+    [super dealloc];
+}
+
+@end
+
+bool asst::MacSCKHelper::hasPermission()
+{
+    return CGPreflightScreenCaptureAccess();
+}
+
+bool asst::MacSCKHelper::requestPermission()
+{
+    return CGRequestScreenCaptureAccess();
+}
+
+struct asst::MacSCKHelper::Impl {
+    ~Impl();
+
+    SCStream* m_stream = nil;
+    MacSCKOutput* m_output = nil;
+
+    dispatch_queue_t m_queue = nil;
+
+    bool init(std::string_view bundle_id, std::string_view port, std::pair<int, int> size, std::array<int16_t, 8> rect);
+    bool capture(std::vector<uint8_t>& bgrData) const;
+};
+
+asst::MacSCKHelper::Impl::~Impl()
+{
+    if (m_stream) {
+        [m_stream stopCaptureWithCompletionHandler:nil];
+        [m_stream release];
+        m_stream = nil;
+    }
+
+    [m_output release];
+    m_output = nil;
+
+    if (m_queue) {
+        dispatch_release(m_queue);
+        m_queue = nil;
+    }
+}
+
+bool asst::MacSCKHelper::Impl::init(std::string_view bundle_id, std::string_view port, std::pair<int, int> size, std::array<int16_t, 8> rect)
+{
+    auto sem = dispatch_semaphore_create(0);
+    __block bool result = false;
+
+    const auto handler = ^(SCShareableContent* _Nullable content, NSError* _Nullable error) {
+        if (error) {
+            Log.error("Cannot get shareable content:", error.localizedDescription.UTF8String);
+            dispatch_semaphore_signal(sem);
+            return;
+        }
+
+        SCWindow* targetWindow = nil;
+        for (SCWindow* window in content.windows) {
+            auto bundleID = window.owningApplication.bundleIdentifier.UTF8String;
+            if (bundleID && bundleID != bundle_id) {
+                continue;
+            }
+            auto title = window.title.UTF8String;
+            if (title && std::string_view(title).find(port) != std::string_view::npos) {
+                targetWindow = window;
+                break;
+            }
+        }
+        if (!targetWindow) {
+            Log.error("No window found with bundle ID:", bundle_id, ", port:", port);
+            dispatch_semaphore_signal(sem);
+            return;
+        }
+
+        const auto filter = [[SCContentFilter alloc] initWithDesktopIndependentWindow:targetWindow];
+
+        auto config = [[SCStreamConfiguration alloc] init];
+        config.width = size.first;
+        config.height = size.second;
+        config.pixelFormat = kCVPixelFormatType_32BGRA;
+        config.colorSpaceName = kCGColorSpaceSRGB;
+        config.showsCursor = NO;
+
+        const auto window_height = rect[3];
+        const auto content_width = rect[6];
+        const auto content_height = rect[7];
+
+        const auto titlebar_height = window_height - content_height;
+        if (titlebar_height > 0) {
+            Log.trace("Titlebar logical height:", titlebar_height);
+            config.sourceRect = CGRectMake(0, titlebar_height, content_width, content_height);
+        }
+
+        m_output = [[MacSCKOutput alloc] init];
+        m_stream = [[SCStream alloc] initWithFilter:filter configuration:config delegate:m_output];
+
+        [config release];
+        [filter release];
+
+        m_queue = dispatch_queue_create("plus.maa.MacSCKOutput", NULL);
+        [m_stream addStreamOutput:m_output
+                             type:SCStreamOutputTypeScreen
+               sampleHandlerQueue:m_queue
+                            error:&error];
+
+        if (error) {
+            Log.error("Cannot add stream output:", error.localizedDescription.UTF8String);
+            dispatch_semaphore_signal(sem);
+            return;
+        }
+
+        [m_stream startCaptureWithCompletionHandler:^(NSError* _Nullable error) {
+            if (error) {
+                Log.error("Cannot start capture:", error.localizedDescription.UTF8String);
+                dispatch_semaphore_signal(sem);
+                return;
+            }
+
+            Log.trace("Started capture for window:", targetWindow.title.UTF8String);
+            m_output.running = YES;
+            result = true;
+            dispatch_semaphore_signal(sem);
+        }];
+    };
+
+    [SCShareableContent getShareableContentExcludingDesktopWindows:YES
+                                               onScreenWindowsOnly:YES
+                                                 completionHandler:handler];
+
+    dispatch_time_t timeout = dispatch_time(DISPATCH_TIME_NOW, 3 * NSEC_PER_SEC);
+    if (dispatch_semaphore_wait(sem, timeout) != 0) {
+        Log.error("Screenshot init timed out.");
+        result = false;
+    }
+
+    dispatch_release(sem);
+    return result;
+}
+
+bool asst::MacSCKHelper::Impl::capture(std::vector<uint8_t>& bgrData) const
+{
+    auto sem = dispatch_semaphore_create(0);
+    __block bool result = false;
+
+    if (!m_queue || !m_output) {
+        Log.error("Stream output is not initialized");
+        dispatch_release(sem);
+        return false;
+    }
+
+    dispatch_async(m_queue, ^{
+        if (!m_output.running) {
+            Log.error("Stream is not running");
+            dispatch_semaphore_signal(sem);
+            return;
+        }
+
+        auto buffer = m_output.buffer;
+        if (!buffer) {
+            Log.error("No image buffer available");
+            dispatch_semaphore_signal(sem);
+            return;
+        }
+
+        long ret = CVPixelBufferLockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly);
+        if (ret != kCVReturnSuccess) [[unlikely]] {
+            Log.error("Failed to lock pixel buffer:", ret);
+            dispatch_semaphore_signal(sem);
+            return;
+        }
+
+        const auto width = CVPixelBufferGetWidth(buffer);
+        const auto height = CVPixelBufferGetHeight(buffer);
+
+        const auto dstRowBytes = width * 3;
+        bgrData.resize(height * dstRowBytes);
+
+        vImage_Buffer srcBuffer;
+        srcBuffer.data = CVPixelBufferGetBaseAddress(buffer);
+        srcBuffer.height = height;
+        srcBuffer.width = width;
+        srcBuffer.rowBytes = CVPixelBufferGetBytesPerRow(buffer);
+
+        vImage_Buffer dstBuffer;
+        dstBuffer.data = bgrData.data();
+        dstBuffer.height = height;
+        dstBuffer.width = width;
+        dstBuffer.rowBytes = dstRowBytes;
+
+        ret = vImageConvert_RGBA8888toRGB888(&srcBuffer, &dstBuffer, kvImageNoFlags);
+        if (ret != kvImageNoError) [[unlikely]] {
+            Log.error("Failed to convert buffer channels:", ret);
+            CVPixelBufferUnlockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly);
+            dispatch_semaphore_signal(sem);
+            return;
+        }
+
+        ret = CVPixelBufferUnlockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly);
+        if (ret != kCVReturnSuccess) [[unlikely]] {
+            Log.error("Failed to unlock pixel buffer:", ret);
+            dispatch_semaphore_signal(sem);
+            return;
+        }
+
+        result = true;
+        dispatch_semaphore_signal(sem);
+    });
+
+    dispatch_time_t timeout = dispatch_time(DISPATCH_TIME_NOW, 3 * NSEC_PER_SEC);
+    if (dispatch_semaphore_wait(sem, timeout) != 0) {
+        Log.error("Screenshot operation timed out.");
+        result = false;
+    }
+
+    dispatch_release(sem);
+    return result;
+}
+
+asst::MacSCKHelper::MacSCKHelper()
+    : pImpl(std::make_unique<Impl>())
+{
+}
+
+asst::MacSCKHelper::~MacSCKHelper() = default;
+
+bool asst::MacSCKHelper::init(std::string_view bundle_id, std::string_view port, std::pair<int, int> size, std::array<int16_t, 8> rect)
+{
+    LogTraceFunction;
+    return pImpl->init(bundle_id, port, size, rect);
+}
+
+bool asst::MacSCKHelper::capture(std::vector<uint8_t>& bgrData) const
+{
+    LogTraceFunction;
+    return pImpl->capture(bgrData);
+}
+
+#endif // ASST_WITH_MAC_SCK

src/MaaCore/Controller/PlayToolsController.cpp

@@ -39,6 +39,10 @@ bool asst::PlayToolsController::connect(
         m_screencap_method = ScreencapMethod::BGR;
         m_minimal_version = 3;
     }
+    else if (config == "MacSCK") {
+        m_screencap_method = ScreencapMethod::MacSCK;
+        m_minimal_version = 3;
+    }
 
     return open();
 }
@@ -72,6 +76,19 @@ bool asst::PlayToolsController::screencap(cv::Mat& image_payload, bool allow_rec
         return screencap_rgba(image_payload, allow_reconnect);
     case ScreencapMethod::BGR:
         return screencap_bgr(image_payload, allow_reconnect);
+    case ScreencapMethod::MacSCK: {
+#if ASST_WITH_MAC_SCK
+        if (!m_sck_helper.capture(m_screencap_buffer)) {
+            return false;
+        }
+        auto mat = cv::Mat(m_screen_size.second, m_screen_size.first, CV_8UC3, m_screencap_buffer.data());
+        mat.copyTo(image_payload);
+        return true;
+#else
+        Log.error("MacSCK is not built, cannot capture screencap with this method");
+        return false;
+#endif // ASST_WITH_MAC_SCK
+    }
     default:
         return false;
     }
@@ -333,7 +350,23 @@ bool asst::PlayToolsController::open()
         return false;
     }
 
-    return check_version() && fetch_screen_res();
+    if (!check_version() || !fetch_screen_res()) {
+        return false;
+    }
+
+    if (m_screencap_method == ScreencapMethod::MacSCK) {
+        if (!fetch_frame_rect() || !fetch_bundle_id()) {
+            return false;
+        }
+#if ASST_WITH_MAC_SCK
+        return m_sck_helper.init(m_bundle_id, port, m_screen_size, m_frame_rect);
+#else
+        Log.error("MacSCK is not built, fallback to BGR screencap method");
+        m_screencap_method = ScreencapMethod::BGR;
+#endif // ASST_WITH_MAC_SCK
+    }
+
+    return true;
 }
 
 bool asst::PlayToolsController::check_version()
@@ -410,3 +443,56 @@ bool asst::PlayToolsController::toucher_commit(const TouchPhase phase, const Poi
     toucher_wait(delay);
     return true;
 }
+
+bool asst::PlayToolsController::fetch_frame_rect()
+{
+    constexpr char request[] = { 0, 4, 'R', 'E', 'C', 'T' };
+
+    try {
+        boost::asio::write(m_socket, boost::asio::buffer(request));
+        boost::asio::read(m_socket, boost::asio::buffer(m_frame_rect));
+    }
+    catch (const std::exception& e) {
+        Log.error("Cannot get frame rectangle:", e.what());
+        return false;
+    }
+
+    for (auto& val : m_frame_rect) {
+        val = socket_ops::network_to_host_short(val);
+    }
+
+    return true;
+}
+
+bool asst::PlayToolsController::fetch_bundle_id()
+{
+    uint32_t length = 0;
+
+    constexpr char request[] = { 0, 4, 'B', 'N', 'D', 'L' };
+
+    try {
+        boost::asio::write(m_socket, boost::asio::buffer(request));
+        boost::asio::read(m_socket, boost::asio::buffer(&length, sizeof(length)));
+        length = socket_ops::network_to_host_long(length);
+    }
+    catch (const std::exception& e) {
+        Log.error("Cannot get bundle ID length:", e.what());
+        return false;
+    }
+
+    if (length == 0 || length > BUFSIZ) {
+        Log.error("Invalid bundle ID length:", length);
+        return false;
+    }
+
+    try {
+        m_bundle_id.resize(length);
+        boost::asio::read(m_socket, boost::asio::buffer(m_bundle_id.data(), length));
+    }
+    catch (const std::exception& e) {
+        Log.error("Cannot get bundle ID:", e.what());
+        return false;
+    }
+
+    return true;
+}

src/MaaCore/Controller/PlayToolsController.h

@@ -10,6 +10,7 @@
 
 #include "Common/AsstMsg.h"
 #include "InstHelper.h"
+#include "MacSCKHelper.h"
 
 namespace asst
 {
@@ -76,6 +77,7 @@ protected:
     {
         RGBA,
         BGR,
+        MacSCK,
     } m_screencap_method = ScreencapMethod::RGBA;
 
     enum class TouchPhase
@@ -96,6 +98,9 @@ protected:
 
 private:
     unsigned m_minimal_version = 2;
+    std::string m_bundle_id;
+    std::array<int16_t, 8> m_frame_rect = { 0, 0, 0, 0, 0, 0, 0, 0 };
+
     void close();
     bool open();
     bool check_version();
@@ -104,5 +109,12 @@ private:
 
     bool screencap_rgba(cv::Mat& image_payload, bool allow_reconnect);
     bool screencap_bgr(cv::Mat& image_payload, bool allow_reconnect);
+
+    bool fetch_frame_rect();
+    bool fetch_bundle_id();
+
+#if ASST_WITH_MAC_SCK
+    MacSCKHelper m_sck_helper;
+#endif // ASST_WITH_MAC_SCK
 };
 } // namespace asst