Fill Suno fields with accessibility adapter

This commit is contained in:
diyaa
2026-09-14 22:29:53 +02:00
parent 448d390671
commit 3fced12bef
6 changed files with 229 additions and 15 deletions
@@ -0,0 +1,111 @@
import AppKit
import ApplicationServices
import MusicAssistantCore
struct MacOSSunoAccessibilityFieldFiller: SunoFieldFilling {
func fill(_ payload: SunoHandoffPayload) async -> SunoFieldFillReport {
let options = [kAXTrustedCheckOptionPrompt.takeUnretainedValue() as String: true] as CFDictionary
guard AXIsProcessTrustedWithOptions(options) else {
return .copyRequired(
for: SunoHandoffField.allCases,
reason: "Allow Accessibility access to fill this field automatically."
)
}
try? await Task.sleep(for: .seconds(2))
guard let application = NSWorkspace.shared.frontmostApplication else {
return .copyRequired(for: SunoHandoffField.allCases, reason: "The Suno browser window was not found.")
}
let processIdentifier = application.processIdentifier
pressCustomMode(in: accessibilityElements(for: processIdentifier))
try? await Task.sleep(for: .milliseconds(500))
let elements = accessibilityElements(for: processIdentifier)
return SunoFieldFillReport(results: Dictionary(uniqueKeysWithValues: SunoHandoffField.allCases.map { field in
(field, fill(field, with: payload.value(for: field), in: elements))
}))
}
private func pressCustomMode(in elements: [AXUIElement]) {
guard let customMode = elements.first(where: { element in
role(of: element) == kAXButtonRole as String && labels(for: element).contains { $0.localizedCaseInsensitiveContains("custom") }
}) else {
return
}
AXUIElementPerformAction(customMode, kAXPressAction as CFString)
}
private func fill(_ field: SunoHandoffField, with value: String, in elements: [AXUIElement]) -> SunoFieldFillResult {
guard !value.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else {
return .filled
}
guard let element = elements.first(where: { matches(field, element: $0) }) else {
return .copyRequired("The \(field.displayName) field was not found.")
}
var isSettable = DarwinBoolean(false)
guard AXUIElementIsAttributeSettable(element, kAXValueAttribute as CFString, &isSettable) == .success,
isSettable.boolValue,
AXUIElementSetAttributeValue(element, kAXValueAttribute as CFString, value as CFTypeRef) == .success else {
return .copyRequired("The \(field.displayName) field could not be filled.")
}
return .filled
}
private func matches(_ field: SunoHandoffField, element: AXUIElement) -> Bool {
let role = role(of: element)
guard role == kAXTextFieldRole as String || role == kAXTextAreaRole as String else { return false }
let labels = labels(for: element)
switch field {
case .title:
return labels.contains { $0.localizedCaseInsensitiveContains("title") }
case .lyrics:
return labels.contains { $0.localizedCaseInsensitiveContains("lyrics") }
case .stylePrompt:
return labels.contains { label in
label.localizedCaseInsensitiveContains("style") || label.localizedCaseInsensitiveContains("prompt")
}
}
}
private func accessibilityElements(for processIdentifier: pid_t) -> [AXUIElement] {
descendants(of: AXUIElementCreateApplication(processIdentifier))
}
private func descendants(of root: AXUIElement) -> [AXUIElement] {
var elements: [AXUIElement] = []
var pending = [root]
while let element = pending.popLast() {
elements.append(element)
pending.append(contentsOf: children(of: element))
}
return elements
}
private func children(of element: AXUIElement) -> [AXUIElement] {
attribute(kAXChildrenAttribute, from: element) as? [AXUIElement] ?? []
}
private func role(of element: AXUIElement) -> String? {
attribute(kAXRoleAttribute, from: element) as? String
}
private func labels(for element: AXUIElement) -> [String] {
[kAXTitleAttribute, kAXDescriptionAttribute, kAXHelpAttribute, kAXIdentifierAttribute]
.compactMap { attribute($0, from: element) as? String }
}
private func attribute(_ name: String, from element: AXUIElement) -> Any? {
var value: CFTypeRef?
guard AXUIElementCopyAttributeValue(element, name as CFString, &value) == .success else { return nil }
return value
}
}
@@ -4,11 +4,20 @@ import SwiftUI
struct FinalReviewView: View { struct FinalReviewView: View {
@Binding var project: SongProject @Binding var project: SongProject
@Environment(\.openURL) private var openURL @Environment(\.openURL) private var openURL
private let sunoFieldFiller: any SunoFieldFilling
@State private var newFieldKey = "" @State private var newFieldKey = ""
@State private var newFieldValue = "" @State private var newFieldValue = ""
@State private var handoffStatus: HandoffStatus? @State private var handoffStatus: HandoffStatus?
init(
project: Binding<SongProject>,
sunoFieldFiller: any SunoFieldFilling = MacOSSunoAccessibilityFieldFiller()
) {
_project = project
self.sunoFieldFiller = sunoFieldFiller
}
var body: some View { var body: some View {
VStack(alignment: .leading, spacing: 12) { VStack(alignment: .leading, spacing: 12) {
ArabicLyricsReviewView(project: $project) ArabicLyricsReviewView(project: $project)
@@ -196,21 +205,37 @@ struct FinalReviewView: View {
} }
openURL(SunoHandoffDestination.createPageURL) { accepted in openURL(SunoHandoffDestination.createPageURL) { accepted in
handoffStatus = accepted guard accepted else {
? .opened handoffStatus = .failed("Unable to open the Suno Create page.")
: .failed("Unable to open the Suno Create page.") return
}
let payload = SunoHandoffPayload(
title: project.title,
lyrics: output.lyricsText,
stylePrompt: output.stylePrompt
)
Task {
handoffStatus = .fieldFill(await sunoFieldFiller.fill(payload))
}
} }
} }
} }
private enum HandoffStatus: Equatable { private enum HandoffStatus: Equatable {
case opened case fieldFill(SunoFieldFillReport)
case failed(String) case failed(String)
var title: String { var title: String {
switch self { switch self {
case .opened: case let .fieldFill(report):
return "Suno Create opened in your browser." let fieldsToCopy = report.results.compactMap { field, result in
if case .copyRequired = result { return field.displayName }
return nil
}
return fieldsToCopy.isEmpty
? "Suno fields filled in your browser."
: "Copy these fields into Suno: \(fieldsToCopy.joined(separator: ", "))."
case let .failed(message): case let .failed(message):
return message return message
} }
@@ -218,8 +243,10 @@ private enum HandoffStatus: Equatable {
var systemImage: String { var systemImage: String {
switch self { switch self {
case .opened: case let .fieldFill(report):
return "checkmark.circle.fill" return report.results.values.contains { if case .copyRequired = $0 { return true }; return false }
? "exclamationmark.triangle.fill"
: "checkmark.circle.fill"
case .failed: case .failed:
return "exclamationmark.triangle.fill" return "exclamationmark.triangle.fill"
} }
@@ -227,8 +254,10 @@ private enum HandoffStatus: Equatable {
var foregroundStyle: Color { var foregroundStyle: Color {
switch self { switch self {
case .opened: case let .fieldFill(report):
return .green return report.results.values.contains { if case .copyRequired = $0 { return true }; return false }
? .orange
: .green
case .failed: case .failed:
return .red return .red
} }
@@ -7,3 +7,66 @@ public protocol SunoHandoffService: Sendable {
public enum SunoHandoffDestination { public enum SunoHandoffDestination {
public static let createPageURL = URL(string: "https://suno.com/create")! public static let createPageURL = URL(string: "https://suno.com/create")!
} }
public struct SunoHandoffPayload: Equatable, Sendable {
public let title: String
public let lyrics: String
public let stylePrompt: String
public init(title: String, lyrics: String, stylePrompt: String) {
self.title = title
self.lyrics = lyrics
self.stylePrompt = stylePrompt
}
public func value(for field: SunoHandoffField) -> String {
switch field {
case .title:
return title
case .lyrics:
return lyrics
case .stylePrompt:
return stylePrompt
}
}
}
public enum SunoHandoffField: String, CaseIterable, Equatable, Identifiable, Sendable {
case title
case lyrics
case stylePrompt
public var id: String { rawValue }
public var displayName: String {
switch self {
case .title:
return "Title"
case .lyrics:
return "Lyrics"
case .stylePrompt:
return "Style prompt"
}
}
}
public enum SunoFieldFillResult: Equatable, Sendable {
case filled
case copyRequired(String)
}
public struct SunoFieldFillReport: Equatable, Sendable {
public let results: [SunoHandoffField: SunoFieldFillResult]
public init(results: [SunoHandoffField: SunoFieldFillResult]) {
self.results = results
}
public static func copyRequired(for fields: [SunoHandoffField], reason: String) -> SunoFieldFillReport {
SunoFieldFillReport(results: Dictionary(uniqueKeysWithValues: fields.map { ($0, .copyRequired(reason)) }))
}
}
public protocol SunoFieldFilling: Sendable {
func fill(_ payload: SunoHandoffPayload) async -> SunoFieldFillReport
}
@@ -9,4 +9,12 @@ final class SunoHandoffDestinationTests: XCTestCase {
XCTAssertEqual(url.host, "suno.com") XCTAssertEqual(url.host, "suno.com")
XCTAssertEqual(url.path, "/create") XCTAssertEqual(url.path, "/create")
} }
func testPayloadPreservesApprovedValuesForEachSupportedField() {
let payload = SunoHandoffPayload(title: "My Song", lyrics: "Approved lyrics", stylePrompt: "Cinematic folk")
XCTAssertEqual(payload.value(for: .title), "My Song")
XCTAssertEqual(payload.value(for: .lyrics), "Approved lyrics")
XCTAssertEqual(payload.value(for: .stylePrompt), "Cinematic folk")
}
} }
+4 -1
View File
@@ -89,7 +89,10 @@ in Custom mode. The integration attempts to fill the supported title,
lyrics and style fields but does not press Generate. If a field cannot lyrics and style fields but does not press Generate. If a field cannot
be filled, it returns a field-specific result so Presentation can report be filled, it returns a field-specific result so Presentation can report
it and offer the approved text for copying. Treat this integration as it and offer the approved text for copying. Treat this integration as
replaceable because website UI/behavior can change. replaceable because website UI/behavior can change. The macOS
Accessibility-based field filler is an adapter behind a protocol so it
can be replaced without changing SongProject, prompt compilation or UI
review logic.
## Instrument Selector Boundary ## Instrument Selector Boundary
+4 -4
View File
@@ -109,11 +109,11 @@ requirement is missing and blocks implementation, record it in
- [x] Implement explicit `Send to Suno` action. - [x] Implement explicit `Send to Suno` action.
- [x] Open the appropriate Suno creation surface. - [x] Open the appropriate Suno creation surface.
- [ ] Fill supported fields with approved project output. - [x] Fill supported fields with approved project output.
- [ ] Never trigger Generate automatically. - [x] Never trigger Generate automatically.
- [ ] Detect/report when fields cannot be filled rather than silently - [x] Detect/report when fields cannot be filled rather than silently
failing. failing.
- [ ] Keep integration isolated because Suno UI can change. - [x] Keep integration isolated because Suno UI can change.
## Phase 9 --- Quality ## Phase 9 --- Quality