--- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -45,7 +45,9 @@ enable_testing() option(SMITH_BUILD_DESKTOP "Build the native Wayfire/PipeWire desktop client" OFF) if(SMITH_BUILD_DESKTOP) - find_package(Qt6 6.11 REQUIRED COMPONENTS Widgets Network Multimedia Test) + find_package(PkgConfig REQUIRED) + pkg_check_modules(DESKTOP REQUIRED IMPORTED_TARGET gtk4>=4.22.5 libportal-gtk4>=0.9 + gstreamer-app-1.0 gstreamer-video-1.0) endif() add_subdirectory(src) add_subdirectory(tests) --- a/packaging/desktop.md +++ b/packaging/desktop.md @@ -1,8 +1,8 @@ # Smith desktop -The optional `smith-desktop` executable is a native Qt 6.11 client for the same +The optional `smith-desktop` executable is a native GTK4 client for the same Smith server, AIRouter sessions and local tool journal as the terminal client. -Build it with `-DSMITH_BUILD_DESKTOP=ON`, or enable `USE=desktop` with the 0.16.1 +Build it with `-DSMITH_BUILD_DESKTOP=ON`, or enable `USE=desktop` with the 0.16.1-r1 candidate recipe. The terminal client and headless server remain available with the option disabled. @@ -10,8 +10,9 @@ Run the launcher as the logged-in desktop user. It selects native Wayland and uses that session's `WAYFIRE_SOCKET`, session bus, PipeWire and screen-cast -portal. The package requires Qt Multimedia with `pipewire`, `ffmpeg` and `dbus`, -Qt Wayland, PipeWire, XDG Desktop Portal and the wlroots portal backend. The +portal. The package requires GTK4, libportal with its GTK4 backend, GStreamer, +PipeWire with its GStreamer plugin, XDG Desktop Portal and the wlroots portal +backend. The desktop binary has no Qt dependency. The session must start the appropriate portal and audio/session-manager services. The supported screen sources depend on that backend; do not infer per-window capture support from Wayfire's window list. @@ -43,7 +44,8 @@ The window controls are user-operated in this version; there is no new agent-facing compositor control API. -Share screen invokes Qt's Wayland ScreenCast/PipeWire path and the system picker. +Share screen invokes libportal's ScreenCast interface and the system picker, +then consumes the selected PipeWire stream through GStreamer. The live preview remains local. Attach snapshot saves a private JPEG under `WORKSPACE/.smith/captures/` and adds its relative path to the editable message. Sending is a separate action. The model's `read_image` returns bounded PNG/JPEG @@ -51,7 +53,7 @@ is resized/encoded below that tool's 700 KiB input limit; encoded results remain under the existing 1 MiB result limit. -Microphone capture uses Qt's PipeWire audio backend, writes a private WAV in the +Microphone capture uses GStreamer's PipeWire source, writes a private WAV in the same capture directory, and stops after 60 seconds or on request. Recordings stay local. Speech transcription, speech synthesis and automatic audio submission are not implemented by this client. They require an explicit AIRouter-owned speech @@ -73,18 +75,34 @@ ## Verification and preview -The Qt IPC fixture exercises fragmented replies, oversized-reply denial and absent +The GIO IPC fixture exercises fragmented replies, oversized-reply denial and absent session handling, plus refusal before login when local bootstrap is missing. A cross-process fake-provider test carries real PNG bytes through the client journal, Smith server and MCP relay. No paid or authenticated model call is needed. ```sh -QT_QPA_PLATFORM=offscreen smith-desktop --preview -QT_QPA_PLATFORM=offscreen smith-desktop --screenshot preview.png -QT_QPA_PLATFORM=offscreen smith-desktop --smoke-test +smith-desktop --preview +smith-desktop --screenshot preview.png +smith-desktop --smoke-test ``` These modes make no Smith connection, model call or capture request. They are interface checks, not live Wayfire/PipeWire qualification. A real user desktop must separately verify the portal picker, screen selection, microphone device, stop/cancel behavior, and an owner-authorized model image turn. + +## GTK4 appearance + +Wayfire/Pixdecor owns the outer frame. The GTK window disables its decorations +and adds no custom compositor titlebar. App headings and controls remain inside. +Contents follow wf-shell's saved `~/.config/wf-shell/appearance.json`: palette, +interface font, text scaling, density, spacing, inset radii, treatment and control +states. Atomic appearance-file replacements are observed while the app runs. +Invalid appearance input retains the last valid style. Wallpaper-derived live +palette previews are not read from the shell resident in this port. + +The GTK4 tests use a private headless Wayland compositor, without changing the +logged-in desktop. Synthetic GStreamer video/audio streams verify frame conversion, +private bounded JPEG snapshots and WAV finalization; they do not access a real +screen or microphone. The real portal picker and microphone device still need a +user-operated check in the desktop session. No model call is part of these checks. --- a/packaging/smith-desktop.desktop +++ b/packaging/smith-desktop.desktop @@ -2,7 +2,7 @@ Type=Application Name=Smith Comment=Local AI conversations, Wayfire controls and PipeWire capture -Exec=env QT_QPA_PLATFORM=wayland smith-desktop +Exec=smith-desktop TryExec=smith-desktop Icon=utilities-terminal Terminal=false --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -58,9 +58,9 @@ if(SMITH_BUILD_DESKTOP) add_library(smith_desktop STATIC smith/desktop/backend.cpp smith/desktop/backend.hpp smith/desktop/wayfire.cpp smith/desktop/wayfire.hpp smith/desktop/media.cpp smith/desktop/media.hpp - smith/desktop/window.cpp smith/desktop/window.hpp) - set_target_properties(smith_desktop PROPERTIES AUTOMOC ON) - target_link_libraries(smith_desktop PUBLIC smith_client Qt6::Widgets Qt6::Network Qt6::Multimedia) + smith/desktop/window.cpp smith/desktop/window.hpp smith/desktop/appearance.cpp + smith/desktop/capture-files.cpp smith/desktop/media-pipeline.cpp) + target_link_libraries(smith_desktop PUBLIC smith_client PkgConfig::DESKTOP) add_executable(smith-desktop smith/desktop/main.cpp) target_link_libraries(smith-desktop PRIVATE smith_desktop) install(TARGETS smith-desktop RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}) --- /dev/null +++ b/src/smith/desktop/appearance.cpp @@ -0,0 +1,91 @@ +#include "smith/desktop/appearance.hpp" +#include "smith/core/json.hpp" +#include +#include +#include +#include +#include + +namespace smith::desktop { +namespace { +std::string quoted(const std::string& text) { + std::string out = "\""; + for (const auto c : text) { + if (static_cast(c) < 32) continue; + if (c == '\\' || c == '"') out += '\\'; + out += c; + } + return out + "\""; +} +std::string css(const json& snapshot) { + const auto t = snapshot.value("theme", json::object()); + const auto palette = t.value("palette", json::object()); + auto color = [&](const char* key, const char* fallback) { + const auto value = palette.value(key, std::string(fallback)); + if (!std::regex_match(value, std::regex("#[0-9a-fA-F]{6}([0-9a-fA-F]{2})?"))) throw std::runtime_error("Invalid appearance color"); + return value; + }; + const auto bg = color("background", "#100f0f"), surface = color("surface", "#1c1b1a"); + const auto fg = color("foreground", "#cecdc3"), muted = color("muted", "#878580"); + const auto accent = color("accent", "#4385be"), on = color("on_accent", "#100f0f"), border = color("border", "#282726"); + const auto metrics = t.value("metrics", json::object()), sizing = snapshot.value("sizing", json::object()); + auto factor = [&](const char* name) { return std::clamp(sizing.value(name, 1.0), 0.5, 3.0); }; + const int font = std::clamp(int(std::lround(metrics.value("font_size", 18) * factor("text_scale"))), 10, 50); + const int gap = std::clamp(int(std::lround(metrics.value("spacing", 8) * factor("density"))), 4, 28); + const int radius = std::clamp(metrics.value("radius", 8), 0, 24); + const auto inputs = t.value("inputs", json::object()); + const int focus = std::clamp(inputs.value("border_width", 2), 1, 6); + const double hover = std::clamp(t.value("surfaces", json::object()).value("hover", 0.1), 0.0, 1.0); + const auto treatment = t.value("treatment", json::object()); + const double depth = treatment.value("style", std::string("soft")) == "flat" ? 0.0 : std::clamp(treatment.value("depth", 0.5), 0.0, 1.0); + std::ostringstream s; s.imbue(std::locale::classic()); + s << "window.smith { background: " << bg << "; color: " << fg << "; border: none; border-radius: 0; box-shadow: none; font-family: " + << quoted(t.value("font_family", std::string("Sans"))) << "; font-size: " << font << "px; }\n" + << ".smith .content { padding: " << gap * 2 << "px; }\n" + << ".smith .heading { font-weight: 600; font-size: 1.2em; } .smith .section-heading { font-weight: 600; }\n" + << ".smith .dim-label { color: " << muted << "; font-size: 0.82em; }\n" + << ".smith .inset { background-color: " << surface << "; border: 1px solid alpha(" << border << ",0.6); border-radius: " + << std::max(0, radius - 2) << "px; padding: " << gap << "px; background-image: linear-gradient(to bottom,alpha(#ffffff," + << depth * .04 << "),alpha(#000000," << depth * .08 << ")); }\n" + << ".smith button, .smith dropdown { color: " << fg << "; background-color: " << surface + << "; background-image: none; box-shadow: none; border: 1px solid alpha(" << border << ",0.4); border-radius: " << radius + << "px; min-height: 28px; padding: " << gap / 2 << "px " << gap << "px; }\n" + << ".smith button:hover, .smith row:hover { background-color: alpha(" << accent << "," << hover << "); }\n" + << ".smith button:disabled { color: " << muted << "; }\n" + << ".smith button:checked, .smith button.suggested-action, .smith row:selected { background: " << accent << "; color: " << on << "; }\n" + << ".smith button.suggested-action:disabled { background: " << surface << "; color: " << muted << "; }\n" + << ".smith button:focus-visible, .smith textview:focus-within, .smith entry:focus-within, .smith dropdown:focus-visible { outline: " + << focus << "px solid " << accent << "; outline-offset: 1px; }\n" + << ".smith textview, .smith textview text, .smith list, .smith entry { background: " << surface << "; color: " << fg << "; }\n" + << ".smith textview { padding: " << gap << "px; } .smith row { padding: " << gap << "px; }\n" + << ".smith selection { background: " << accent << "; color: " << on << "; }\n"; + return s.str(); +} +} +Appearance::Appearance(GdkDisplay* display) : display_(display), path_(std::string(g_get_user_config_dir()) + "/wf-shell/appearance.json") { + reload(); + auto* file = g_file_new_for_path((std::string(g_get_user_config_dir()) + "/wf-shell").c_str()); + monitor_ = g_file_monitor_directory(file, G_FILE_MONITOR_WATCH_MOVES, nullptr, nullptr); g_object_unref(file); + if (monitor_) g_signal_connect(monitor_, "changed", G_CALLBACK(+[](GFileMonitor*, GFile*, GFile*, GFileMonitorEvent, gpointer p) { + static_cast(p)->reload(); + }), this); +} +Appearance::~Appearance() { + if (monitor_) { g_file_monitor_cancel(monitor_); g_object_unref(monitor_); } + if (provider_) { gtk_style_context_remove_provider_for_display(display_, GTK_STYLE_PROVIDER(provider_)); g_object_unref(provider_); } +} +void Appearance::reload() { + try { + json snapshot = json::object(); std::ifstream input(path_); + if (input) { + input.seekg(0, std::ios::end); if (input.tellg() > 1024 * 1024) throw std::runtime_error("Appearance file is too large"); + input.seekg(0); input >> snapshot; + } + const auto text = css(snapshot); + auto* next = gtk_css_provider_new(); gtk_css_provider_load_from_string(next, text.c_str()); + gtk_style_context_add_provider_for_display(display_, GTK_STYLE_PROVIDER(next), GTK_STYLE_PROVIDER_PRIORITY_APPLICATION); + if (provider_) { gtk_style_context_remove_provider_for_display(display_, GTK_STYLE_PROVIDER(provider_)); g_object_unref(provider_); } + provider_ = next; + } catch (const std::exception& e) { g_printerr("Smith appearance: %s\n", e.what()); } +} +} --- /dev/null +++ b/src/smith/desktop/appearance.hpp @@ -0,0 +1,16 @@ +#pragma once +#include +#include +namespace smith::desktop { +class Appearance { +public: + explicit Appearance(GdkDisplay* display); + ~Appearance(); + void reload(); +private: + GdkDisplay* display_; + GtkCssProvider* provider_ = nullptr; + GFileMonitor* monitor_ = nullptr; + std::string path_; +}; +} --- a/src/smith/desktop/backend.cpp +++ b/src/smith/desktop/backend.cpp @@ -1,6 +1,5 @@ #include "smith/desktop/backend.hpp" #include -#include #include #include #include @@ -17,13 +16,13 @@ std::optional session; bool running = false, stopping = false; // io-owned: settle cancellation before a new turn }; -Backend::Backend(QObject* parent) : QObject(parent), state_(std::make_unique()) { +Backend::Backend(Callbacks callbacks) : state_(std::make_unique()), callbacks_(std::move(callbacks)) { state_->thread = std::thread([this] { state_->io.run(); }); } Backend::~Backend() { - state_->guard.reset(); state_->io.stop(); state_->thread.join(); + *alive_ = false; state_->guard.reset(); state_->io.stop(); state_->thread.join(); } -void Backend::open(QString workspace, QString session) { +void Backend::open(std::string workspace, std::string session) { auto& s = *state_; auto owns = std::make_shared(false); asio::co_spawn(s.io, [this, workspace, session, owns]() -> asio::awaitable { @@ -38,8 +37,8 @@ state.connection = std::make_shared(state.io.get_executor(), std::move(settings)); auto connection = state.connection; client::chat_options options; - options.workspace = workspace.toStdString(); options.name = "Desktop"; options.grants = {false, false, false}; - if (!session.isEmpty()) options.session = session.toStdString(); + options.workspace = workspace; options.name = "Desktop"; options.grants = {false, false, false}; + if (!session.empty()) options.session = session; state.session = co_await client::open_session(*connection, options); require(state.session->grants == grants{false, false, false}, "This first desktop client opens read-only workspace sessions"); state.executor = std::make_shared(state.session->workspace, @@ -48,7 +47,7 @@ for (const auto& event : state.session->history) { const auto e = client::session_event(event); if (e.type == client::drive_event::user || e.type == client::drive_event::answer) - Q_EMIT line(e.type == client::drive_event::user ? "You" : "Smith", QString::fromStdString(e.text)); + emit(callbacks_.line, e.type == client::drive_event::user ? "You" : "Smith", e.text); } request list; list.type = "sessions"; const auto available = co_await connection->rpc(list); @@ -58,25 +57,25 @@ grants::from_json(item.at("grants"), "session.grants") == grants{false, false, false}) local.push_back(item); } - Q_EMIT sessions(QJsonDocument::fromJson(QByteArray::fromStdString(local.dump())).array()); - Q_EMIT ready(QString::fromStdString(state.session->id)); + emit(callbacks_.sessions, local); + emit(callbacks_.ready, state.session->id); if (state.session->active) { - Q_EMIT running(); + emit(callbacks_.running); request advance; advance.type = "advance"; advance.session = state.session->id; auto initial = co_await connection->rpc(advance); auto out = co_await client::drive(*connection, *executor, state.session->id, initial, state.session->events_after, [this](const client::drive_event& e) { - if (e.type == client::drive_event::answer) Q_EMIT line("Smith", QString::fromStdString(e.text)); - if (e.type == client::drive_event::phase) Q_EMIT phase(QString::fromStdString(e.text)); + if (e.type == client::drive_event::answer) emit(callbacks_.line, "Smith", e.text); + if (e.type == client::drive_event::phase) emit(callbacks_.phase, e.text); }); state.session->events_after = out.events_after; } }, [this, owns](std::exception_ptr e) { - if (e) { try { std::rethrow_exception(e); } catch (const std::exception& ex) { Q_EMIT failed(ex.what()); } } - if (*owns) { state_->running = false; if (!state_->stopping) Q_EMIT finished(); } + if (e) { try { std::rethrow_exception(e); } catch (const std::exception& ex) { emit(callbacks_.failed, std::string(ex.what())); } } + if (*owns) { state_->running = false; if (!state_->stopping) emit(callbacks_.finished); } }); } -void Backend::send(QString message) { +void Backend::send(std::string message) { auto owns = std::make_shared(false); asio::co_spawn(state_->io, [this, message, owns]() -> asio::awaitable { auto& s = *state_; @@ -87,21 +86,21 @@ auto executor = s.executor; std::string last; auto out = co_await client::send_message(*connection, *executor, s.session->id, - message.toStdString(), s.session->events_after, [this, &last](const client::drive_event& event) { + message, s.session->events_after, [this, &last](const client::drive_event& event) { using E = client::drive_event; - if (event.type == E::phase) Q_EMIT phase(QString::fromStdString(event.text)); + if (event.type == E::phase) emit(callbacks_.phase, event.text); else if (event.type == E::answer || event.type == E::commentary) { - if (last != event.text) Q_EMIT line("Smith", QString::fromStdString(event.text)); + if (last != event.text) emit(callbacks_.line, "Smith", event.text); last = event.text; } else if (event.type == E::tool_started) { - last.clear(); Q_EMIT line("Tool", QString::fromStdString(event.text)); + last.clear(); emit(callbacks_.line, "Tool", event.text); } }); s.session->events_after = out.events_after; - Q_EMIT phase(QString::fromStdString(out.phase)); + emit(callbacks_.phase, out.phase); }, [this, owns](std::exception_ptr e) { - if (e) { try { std::rethrow_exception(e); } catch (const std::exception& ex) { Q_EMIT failed(ex.what()); } } - if (*owns) { state_->running = false; if (!state_->stopping) Q_EMIT finished(); } + if (e) { try { std::rethrow_exception(e); } catch (const std::exception& ex) { emit(callbacks_.failed, std::string(ex.what())); } } + if (*owns) { state_->running = false; if (!state_->stopping) emit(callbacks_.finished); } }); } void Backend::stop() { @@ -113,8 +112,8 @@ auto connection = s.connection; request r; r.type = "cancel"; r.session = s.session->id; co_await connection->rpc(r); }, [this, owns](std::exception_ptr e) { - if (e) { try { std::rethrow_exception(e); } catch (const std::exception& ex) { Q_EMIT failed(QString("Stop request: ") + ex.what()); } } - if (*owns) { state_->stopping = false; if (!state_->running) Q_EMIT finished(); } + if (e) { try { std::rethrow_exception(e); } catch (const std::exception& ex) { emit(callbacks_.failed, std::string("Stop request: ") + ex.what()); } } + if (*owns) { state_->stopping = false; if (!state_->running) emit(callbacks_.finished); } }); } } --- a/src/smith/desktop/backend.hpp +++ b/src/smith/desktop/backend.hpp @@ -1,28 +1,31 @@ #pragma once -#include -#include -#include -#include +#include "smith/core/json.hpp" +#include "smith/desktop/events.hpp" +#include namespace smith::desktop { -class Backend : public QObject { - Q_OBJECT +class Backend { public: - explicit Backend(QObject* parent = nullptr); - ~Backend() override; - void open(QString workspace, QString session = {}); - void send(QString message); + struct Callbacks { + std::function ready, phase, failed; + std::function sessions; + std::function running, finished; + std::function line; + }; + explicit Backend(Callbacks callbacks); + ~Backend(); + void open(std::string workspace, std::string session = {}); + void send(std::string message); void stop(); -Q_SIGNALS: - void ready(QString session); - void sessions(QJsonArray sessions); - void running(); - void line(QString role, QString text); - void phase(QString text); - void failed(QString error); - void finished(); private: + template void emit(F callback, Args... args) { + if (!callback) return; + auto alive = alive_; + post([alive, callback, args...] { if (*alive) callback(args...); }); + } struct State; std::unique_ptr state_; + Callbacks callbacks_; + std::shared_ptr alive_ = std::make_shared(true); }; } --- /dev/null +++ b/src/smith/desktop/capture-files.cpp @@ -0,0 +1,54 @@ +#include "smith/desktop/capture-files.hpp" +#include +#include +#include +#include +#include +#include +#include +#include + +namespace smith::desktop { +namespace { +void check(bool good, const char* message) { if (!good) throw std::system_error(errno, std::generic_category(), message); } +int child(int parent, const char* name) { + check(::mkdirat(parent, name, 0700) == 0 || errno == EEXIST, "Create capture directory"); + int fd = ::openat(parent, name, O_RDONLY | O_DIRECTORY | O_CLOEXEC | O_NOFOLLOW); + if (fd < 0) throw std::runtime_error("Capture directories must not be symlinks"); + struct stat st{}; + if (::fstat(fd, &st) || st.st_uid != ::geteuid()) { ::close(fd); throw std::runtime_error("Capture directories must be owned by you"); } + if (::fchmod(fd, 0700)) { ::close(fd); throw std::runtime_error("Cannot protect capture directory"); } + return fd; +} +} +CaptureFile::CaptureFile(const std::string& workspace, const std::string& extension) { + int root = ::open(workspace.c_str(), O_RDONLY | O_DIRECTORY | O_CLOEXEC), smith = -1; + try { + check(root >= 0, "Open capture workspace"); smith = child(root, ".smith"); directory = child(smith, "captures"); + auto* uuid = g_uuid_string_random(); name = std::string(uuid) + extension; g_free(uuid); + temporary = "." + name + ".part"; + path = (std::filesystem::path(workspace) / ".smith" / "captures" / name).string(); + fd = ::openat(directory, temporary.c_str(), O_WRONLY | O_CREAT | O_EXCL | O_CLOEXEC | O_NOFOLLOW, 0600); + check(fd >= 0, "Create private capture"); + ::close(smith); ::close(root); + } catch (...) { + if (root >= 0) ::close(root); + if (smith >= 0) ::close(smith); + if (directory >= 0) { ::close(directory); directory = -1; } + throw; + } +} +CaptureFile::~CaptureFile() { + if (fd >= 0) ::close(fd); + if (directory >= 0) { + if (!temporary.empty()) ::unlinkat(directory, temporary.c_str(), 0); + ::close(directory); + } +} +void CaptureFile::commit() { + check(::fsync(fd) == 0, "Flush capture"); + // Publish without replacing any existing name, even in a shared workspace. + check(::linkat(directory, temporary.c_str(), directory, name.c_str(), 0) == 0, "Publish capture"); + ::unlinkat(directory, temporary.c_str(), 0); temporary.clear(); +} +} --- /dev/null +++ b/src/smith/desktop/capture-files.hpp @@ -0,0 +1,13 @@ +#pragma once +#include +namespace smith::desktop { +struct CaptureFile { + int fd = -1, directory = -1; + std::string temporary, name, path; + CaptureFile(const std::string& workspace, const std::string& extension); + ~CaptureFile(); + CaptureFile(const CaptureFile&) = delete; + CaptureFile& operator=(const CaptureFile&) = delete; + void commit(); +}; +} --- /dev/null +++ b/src/smith/desktop/events.hpp @@ -0,0 +1,16 @@ +#pragma once +#include +#include +#include +#include + +namespace smith::desktop { +inline void post(std::function callback) { + g_idle_add_full(G_PRIORITY_DEFAULT, [](gpointer data) -> gboolean { + (*static_cast*>(data))(); + return G_SOURCE_REMOVE; + }, new std::function(std::move(callback)), [](gpointer data) { + delete static_cast*>(data); + }); +} +} --- a/src/smith/desktop/main.cpp +++ b/src/smith/desktop/main.cpp @@ -1,23 +1,54 @@ -#include -#include -#include +#include +#include #include +#include +#include +#include #include "smith/desktop/window.hpp" +namespace { +struct App { + std::unique_ptr window; + bool preview = false, smoke = false; + std::string screenshot; + int result = 0; +}; +bool screenshot(GtkWindow* window, const std::string& path) { + const int width = gtk_widget_get_width(GTK_WIDGET(window)), height = gtk_widget_get_height(GTK_WIDGET(window)); + if (width <= 0 || height <= 0) return false; + auto* paintable = gtk_widget_paintable_new(GTK_WIDGET(window)); auto* snapshot = gtk_snapshot_new(); + gdk_paintable_snapshot(paintable, GDK_SNAPSHOT(snapshot), width, height); g_object_unref(paintable); + auto* node = gtk_snapshot_free_to_node(snapshot); if (!node) return false; + auto* renderer = gtk_native_get_renderer(GTK_NATIVE(window)); + auto* texture = renderer ? gsk_renderer_render_texture(renderer, node, nullptr) : nullptr; gsk_render_node_unref(node); + if (!texture) return false; + const bool saved = gdk_texture_save_to_png(texture, path.c_str()); g_object_unref(texture); return saved; +} +} int main(int argc, char** argv) { - ::umask(0077); - // Both Qt's audio and Wayland screen paths use this session's PipeWire. - qputenv("QT_AUDIO_BACKEND", "pipewire"); - QApplication app(argc, argv); app.setApplicationName("Smith Desktop"); app.setOrganizationName("HedronFire"); - QCommandLineParser args; args.addHelpOption(); - args.addOption({"preview", "Preview the interface without login, model calls or capture."}); - args.addOption({"screenshot", "Save a preview screenshot and exit.", "file"}); - args.addOption({"smoke-test", "Open the preview briefly and exit."}); args.process(app); - const bool preview = args.isSet("preview") || args.isSet("smoke-test") || args.isSet("screenshot"); - smith::desktop::Window window(preview); window.show(); - if (args.isSet("screenshot")) QTimer::singleShot(400, &app, [&] { - app.exit(window.grab().save(args.value("screenshot")) ? 0 : 1); - }); - else if (args.isSet("smoke-test")) QTimer::singleShot(250, &app, &QApplication::quit); - return app.exec(); + ::umask(0077); App state; + for (int i = 1; i < argc; ++i) { + const std::string arg(argv[i]); + if (arg == "--preview") state.preview = true; + else if (arg == "--smoke-test") { state.preview = true; state.smoke = true; } + else if (arg == "--screenshot" && i + 1 < argc) { state.preview = true; state.screenshot = argv[++i]; } + else if (arg == "--help" || arg == "-h") { + std::cout << "Usage: smith-desktop [--preview | --smoke-test | --screenshot FILE]\nNative GTK4 Smith desktop. Preview modes make no model or capture requests.\n"; return 0; + } else { std::cerr << "Unknown or incomplete argument: " << arg << '\n'; return 2; } + } + gst_init(nullptr, nullptr); + auto* app = gtk_application_new("com.hedronfire.Smith", G_APPLICATION_NON_UNIQUE); + g_signal_connect(app, "activate", G_CALLBACK(+[](GtkApplication* app, gpointer raw) { + auto& state = *static_cast(raw); + state.window = std::make_unique(app, state.preview); gtk_window_present(state.window->widget()); + if (state.smoke || !state.screenshot.empty()) { + g_timeout_add(500, [](gpointer raw) -> gboolean { + auto& state = *static_cast(raw); + if (!state.screenshot.empty() && !screenshot(state.window->widget(), state.screenshot)) state.result = 1; + g_application_quit(G_APPLICATION(gtk_window_get_application(state.window->widget()))); return G_SOURCE_REMOVE; + }, &state); + } + }), &state); + const int result = g_application_run(G_APPLICATION(app), 1, argv); state.window.reset(); g_object_unref(app); + return result ? result : state.result; } --- /dev/null +++ b/src/smith/desktop/media-internal.hpp @@ -0,0 +1,26 @@ +#pragma once +#include "smith/desktop/media.hpp" +#include "smith/desktop/capture-files.hpp" +#include +#include +#include +namespace smith::desktop { +struct Media::State { + Callbacks callbacks; + bool alive = true; + unsigned generation = 0; + XdpPortal* portal = nullptr; + XdpSession* session = nullptr; + GCancellable* cancel = nullptr; + gulong closed = 0; + int remote = -1; + GstElement *screen = nullptr, *audio = nullptr; + guint screenBus = 0, audioBus = 0, frames = 0, limit = 0; + GdkPixbuf* image = nullptr; + std::unique_ptr audioFile; + ~State(); +}; +gboolean poll_frame(gpointer state); +void stop_screen(Media::State& state); +void start_pipeline(const std::shared_ptr& state, guint32 node, GVariant* properties); +} --- /dev/null +++ b/src/smith/desktop/media-pipeline.cpp @@ -0,0 +1,126 @@ +#include "smith/desktop/media-internal.hpp" +#include +#include +#include +#include + +namespace smith::desktop { +void stop_screen(Media::State& s) { + ++s.generation; + if (s.cancel) { g_cancellable_cancel(s.cancel); g_clear_object(&s.cancel); } + if (s.frames) { g_source_remove(s.frames); s.frames = 0; } + if (s.screenBus) { g_source_remove(s.screenBus); s.screenBus = 0; } + if (s.screen) { gst_element_set_state(s.screen, GST_STATE_NULL); gst_object_unref(s.screen); s.screen = nullptr; } + if (s.remote >= 0) { ::close(s.remote); s.remote = -1; } + if (s.session) { + auto* session = s.session; s.session = nullptr; + if (s.closed) { g_signal_handler_disconnect(session, s.closed); s.closed = 0; } + xdp_session_close(session); g_object_unref(session); + } + g_clear_object(&s.image); + if (s.alive) { s.callbacks.frame(nullptr); s.callbacks.screenState(false); } +} +gboolean poll_frame(gpointer raw) { + auto& s = *static_cast(raw); + auto* sink = gst_bin_get_by_name(GST_BIN(s.screen), "preview"); + auto* sample = gst_app_sink_try_pull_sample(GST_APP_SINK(sink), 0); gst_object_unref(sink); + if (!sample) return G_SOURCE_CONTINUE; + GstVideoInfo info{}; GstVideoFrame frame{}; + if (gst_video_info_from_caps(&info, gst_sample_get_caps(sample)) && + GST_VIDEO_INFO_FORMAT(&info) == GST_VIDEO_FORMAT_RGBA && + GST_VIDEO_INFO_WIDTH(&info) > 0 && GST_VIDEO_INFO_WIDTH(&info) <= 1600 && + GST_VIDEO_INFO_HEIGHT(&info) > 0 && GST_VIDEO_INFO_HEIGHT(&info) <= 1200 && + gst_video_frame_map(&frame, &info, gst_sample_get_buffer(sample), GST_MAP_READ)) { + auto* image = gdk_pixbuf_new(GDK_COLORSPACE_RGB, true, 8, info.width, info.height); + if (image) { + const auto* pixels = static_cast(GST_VIDEO_FRAME_PLANE_DATA(&frame, 0)); + const auto stride = GST_VIDEO_FRAME_PLANE_STRIDE(&frame, 0); + for (int y = 0; y < info.height; ++y) + std::memcpy(gdk_pixbuf_get_pixels(image) + y * gdk_pixbuf_get_rowstride(image), pixels + static_cast(y) * stride, info.width * 4); + g_clear_object(&s.image); s.image = image; + auto* bytes = g_bytes_new(gdk_pixbuf_get_pixels(image), gdk_pixbuf_get_byte_length(image)); + auto* texture = gdk_memory_texture_new(info.width, info.height, GDK_MEMORY_R8G8B8A8, bytes, gdk_pixbuf_get_rowstride(image)); + g_bytes_unref(bytes); s.callbacks.frame(texture); g_object_unref(texture); + } + gst_video_frame_unmap(&frame); + } + gst_sample_unref(sample); return G_SOURCE_CONTINUE; +} +void start_pipeline(const std::shared_ptr& state, guint32 node, GVariant* properties) { + auto& s = *state; + GError* error = nullptr; + s.screen = gst_parse_launch("pipewiresrc name=source do-timestamp=true ! videoconvert ! videoscale ! " + "video/x-raw,format=RGBA,width=(int)[1,1600],height=(int)[1,1200],pixel-aspect-ratio=1/1 ! " + "appsink name=preview max-buffers=1 drop=true sync=false", &error); + if (error) { const std::string message = error->message; g_error_free(error); throw std::runtime_error(message); } + auto* source = gst_bin_get_by_name(GST_BIN(s.screen), "source"); + guint64 serial = 0; + if (g_variant_lookup(properties, "pipewire-serial", "t", &serial)) + g_object_set(source, "target-object", std::to_string(serial).c_str(), nullptr); + else g_object_set(source, "path", std::to_string(node).c_str(), nullptr); + g_object_set(source, "fd", s.remote, nullptr); gst_object_unref(source); + auto* bus = gst_element_get_bus(s.screen); + s.screenBus = gst_bus_add_watch(bus, [](GstBus*, GstMessage* message, gpointer raw) -> gboolean { + auto& s = *static_cast(raw); + if (GST_MESSAGE_TYPE(message) == GST_MESSAGE_ERROR || GST_MESSAGE_TYPE(message) == GST_MESSAGE_EOS) { + std::string text = "Screen sharing ended"; + if (GST_MESSAGE_TYPE(message) == GST_MESSAGE_ERROR) { + GError* error = nullptr; gst_message_parse_error(message, &error, nullptr); text = error->message; g_error_free(error); + } + s.screenBus = 0; stop_screen(s); if (s.alive) s.callbacks.failed(text); return G_SOURCE_REMOVE; + } + return G_SOURCE_CONTINUE; + }, &s); gst_object_unref(bus); + s.frames = g_timeout_add(100, poll_frame, &s); + if (gst_element_set_state(s.screen, GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) + throw std::runtime_error("Could not start the portal PipeWire stream"); + s.callbacks.screenState(true); +} +void Media::record(const std::string& workspace) { + auto& s = *state_; if (s.audio) return; + try { + s.audioFile = std::make_unique(workspace, ".wav"); + GError* error = nullptr; + s.audio = gst_parse_launch("pipewiresrc do-timestamp=true ! audioconvert ! audioresample ! " + "audio/x-raw,format=S16LE,rate=16000,channels=1 ! wavenc ! fdsink name=recording sync=false", &error); + if (error) { const std::string text = error->message; g_error_free(error); throw std::runtime_error(text); } + auto* sink = gst_bin_get_by_name(GST_BIN(s.audio), "recording"); + g_object_set(sink, "fd", s.audioFile->fd, nullptr); gst_object_unref(sink); + auto* bus = gst_element_get_bus(s.audio); + s.audioBus = gst_bus_add_watch(bus, [](GstBus*, GstMessage* message, gpointer raw) -> gboolean { + auto* self = static_cast(raw); + if (GST_MESSAGE_TYPE(message) == GST_MESSAGE_ERROR) { + self->state_->audioBus = 0; self->state_->callbacks.failed("Microphone capture failed"); + self->stopRecording(); return G_SOURCE_REMOVE; + } + return G_SOURCE_CONTINUE; + }, this); gst_object_unref(bus); + if (gst_element_set_state(s.audio, GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) + throw std::runtime_error("Could not start microphone capture"); + s.limit = g_timeout_add_seconds(60, [](gpointer raw) -> gboolean { + auto* self = static_cast(raw); self->state_->limit = 0; self->stopRecording(); return G_SOURCE_REMOVE; + }, this); + s.callbacks.recordingState(true); + } catch (const std::exception& e) { s.callbacks.failed(e.what()); stopRecording(); } +} +void Media::stopRecording() { + auto& s = *state_; + if (s.limit) { g_source_remove(s.limit); s.limit = 0; } + if (s.audioBus) { g_source_remove(s.audioBus); s.audioBus = 0; } + if (!s.audio) { s.audioFile.reset(); return; } + auto* bus = gst_element_get_bus(s.audio); + gst_element_send_event(s.audio, gst_event_new_eos()); + auto* message = gst_bus_timed_pop_filtered(bus, 2 * GST_SECOND, + static_cast(GST_MESSAGE_EOS | GST_MESSAGE_ERROR)); + const bool complete = message && GST_MESSAGE_TYPE(message) == GST_MESSAGE_EOS; + if (message) gst_message_unref(message); + gst_object_unref(bus); gst_element_set_state(s.audio, GST_STATE_NULL); gst_object_unref(s.audio); s.audio = nullptr; + try { + const auto length = ::lseek(s.audioFile->fd, 0, SEEK_END); + if (!complete || length <= 44 || length > 64 * 1024 * 1024) + throw std::runtime_error("Microphone recording was empty or could not be finalized"); + s.audioFile->commit(); if (s.alive) s.callbacks.saved(s.audioFile->path); + } catch (const std::exception& e) { if (s.alive) s.callbacks.failed(e.what()); } + s.audioFile.reset(); if (s.alive) s.callbacks.recordingState(false); +} +} --- a/src/smith/desktop/media.cpp +++ b/src/smith/desktop/media.cpp @@ -1,111 +1,99 @@ -#include "smith/desktop/media.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include "smith/desktop/media-internal.hpp" +#include #include namespace smith::desktop { namespace { -QString destination(const QString& workspace, const QString& extension) { - QDir directory(workspace); - for (const auto& name : {QString(".smith"), QString(".smith/captures")}) { - QFileInfo info(directory.filePath(name)); - if (info.isSymLink() || (info.exists() && (!info.isDir() || info.ownerId() != ::geteuid()))) - throw std::runtime_error("Capture directory must be an owned directory, without symlinks"); +struct Operation { + std::shared_ptr state; + unsigned generation; + XdpParent* parent; + ~Operation() { if (parent) xdp_parent_free(parent); } + bool current() const { return state->alive && state->generation == generation; } +}; +void fail(Operation& op, GError* error) { + if (op.current()) { + stop_screen(*op.state); + op.state->callbacks.failed(error ? error->message : "Screen sharing was cancelled"); } - if (!directory.mkpath(".smith/captures")) throw std::runtime_error("Cannot create the workspace capture directory"); - const auto folder = directory.filePath(".smith/captures"); - if (!QFile::setPermissions(folder, QFileDevice::ReadOwner | QFileDevice::WriteOwner | QFileDevice::ExeOwner)) - throw std::runtime_error("Cannot protect the capture directory"); - return QDir(folder).filePath(QDateTime::currentDateTimeUtc().toString("yyyyMMdd-HHmmss-zzz") + "-" + - QUuid::createUuid().toString(QUuid::WithoutBraces).left(8) + extension); + g_clear_error(&error); } -QByteArray wavHeader(const QAudioFormat& format, quint32 bytes) { - QByteArray h(44, '\0'); - h.replace(0, 4, "RIFF"); qToLittleEndian(bytes + 36, h.data() + 4); - h.replace(8, 8, "WAVEfmt "); qToLittleEndian(16, h.data() + 16); - qToLittleEndian(format.sampleFormat() == QAudioFormat::Float ? 3 : 1, h.data() + 20); - qToLittleEndian(format.channelCount(), h.data() + 22); - qToLittleEndian(format.sampleRate(), h.data() + 24); - qToLittleEndian(format.sampleRate() * format.bytesPerFrame(), h.data() + 28); - qToLittleEndian(format.bytesPerFrame(), h.data() + 32); - qToLittleEndian(format.bytesPerSample() * 8, h.data() + 34); - h.replace(36, 4, "data"); qToLittleEndian(bytes, h.data() + 40); return h; +void started(GObject* source, GAsyncResult* result, gpointer data) { + std::unique_ptr op(static_cast(data)); + GError* error = nullptr; + const bool good = xdp_session_start_finish(XDP_SESSION(source), result, &error); + if (!op->current()) { g_clear_error(&error); return; } + if (!good) { fail(*op, error); return; } + auto& s = *op->state; + auto* streams = xdp_session_get_streams(s.session); + if (!streams || g_variant_n_children(streams) != 1) { fail(*op, nullptr); return; } + guint32 node; GVariant* properties = nullptr; + g_variant_get_child(streams, 0, "(u@a{sv})", &node, &properties); + s.remote = xdp_session_open_pipewire_remote(s.session); + if (s.remote < 0) { g_variant_unref(properties); fail(*op, nullptr); return; } + try { start_pipeline(op->state, node, properties); } + catch (const std::exception& e) { stop_screen(s); s.callbacks.failed(e.what()); } + g_variant_unref(properties); +} +void created(GObject* source, GAsyncResult* result, gpointer data) { + std::unique_ptr op(static_cast(data)); + GError* error = nullptr; + auto* session = xdp_portal_create_screencast_session_finish(XDP_PORTAL(source), result, &error); + if (!op->current()) { + if (session) { xdp_session_close(session); g_object_unref(session); } + g_clear_error(&error); return; + } + if (!session) { fail(*op, error); return; } + auto& s = *op->state; s.session = session; + s.closed = g_signal_connect(session, "closed", G_CALLBACK(+[](XdpSession*, gpointer raw) { + stop_screen(*static_cast(raw)); + }), &s); + auto* next = op.release(); + xdp_session_start(session, next->parent, s.cancel, started, next); } } -Media::Media(QObject* parent) : QObject(parent) { - capture_.setScreenCapture(&screen_); capture_.setVideoSink(&sink_); - connect(&sink_, &QVideoSink::videoFrameChanged, this, [this](const QVideoFrame& video) { - if (!screen_.isActive()) return; - auto image = video.toImage(); if (image.isNull()) return; - image_ = image.scaled(1600, 1200, Qt::KeepAspectRatio, Qt::SmoothTransformation); Q_EMIT frame(image_); - }); - connect(&screen_, &QScreenCapture::activeChanged, this, &Media::screenState); - connect(&screen_, &QScreenCapture::errorOccurred, this, [this](auto, const QString& error) { - screen_.stop(); image_ = {}; Q_EMIT frame({}); Q_EMIT screenState(false); - Q_EMIT failed("Screen sharing: " + error); - }); - limit_.setSingleShot(true); connect(&limit_, &QTimer::timeout, this, &Media::stopRecording); +Media::State::~State() { g_clear_object(&portal); } +Media::Media(Callbacks callbacks) : state_(std::make_shared()) { state_->callbacks = std::move(callbacks); } +Media::~Media() { state_->alive = false; stopAll(); } +void Media::shareScreen(bool share, GtkWindow* parent) { + auto& s = *state_; + if (!share || !parent) { stop_screen(s); return; } + if (s.screen || s.cancel || s.session) stop_screen(s); + if (!s.portal) s.portal = xdp_portal_new(); + s.cancel = g_cancellable_new(); + auto* op = new Operation{state_, s.generation, xdp_parent_new_gtk(parent)}; + xdp_portal_create_screencast_session(s.portal, XDP_OUTPUT_MONITOR, + XDP_SCREENCAST_FLAG_NONE, XDP_CURSOR_MODE_EMBEDDED, XDP_PERSIST_MODE_NONE, + nullptr, s.cancel, created, op); } -Media::~Media() { stopAll(); } -void Media::shareScreen(bool share) { - if (share) screen_.start(); - else { screen_.stop(); image_ = {}; Q_EMIT frame({}); } +bool Media::recording() const { return state_->audio != nullptr; } +void Media::stopAll() { stop_screen(*state_); stopRecording(); } +std::string Media::snapshot(const std::string& workspace) { + auto& s = *state_; + if (!s.screen || !s.image) { s.callbacks.failed("Share a screen before taking a snapshot"); return {}; } + auto* image = GDK_PIXBUF(g_object_ref(s.image)); + gchar* bytes = nullptr; gsize size = 0; GError* error = nullptr; + try { + for (;;) { + if (!gdk_pixbuf_save_to_buffer(image, &bytes, &size, "jpeg", &error, "quality", "75", nullptr)) + throw std::runtime_error(error ? error->message : "Cannot encode snapshot"); + if (size <= 650 * 1024) break; + g_free(bytes); bytes = nullptr; + const int width = gdk_pixbuf_get_width(image) * 3 / 4, height = gdk_pixbuf_get_height(image) * 3 / 4; + if (width < 160 || height < 1) throw std::runtime_error("Snapshot is too large"); + auto* scaled = gdk_pixbuf_scale_simple(image, width, height, GDK_INTERP_BILINEAR); + g_object_unref(image); image = scaled; + } + CaptureFile file(workspace, ".jpg"); + std::size_t offset = 0; + while (offset < size) { + const auto count = ::write(file.fd, bytes + offset, size - offset); + if (count < 0 && errno == EINTR) continue; + if (count <= 0) throw std::runtime_error("Cannot write snapshot"); + offset += static_cast(count); + } + file.commit(); g_free(bytes); g_object_unref(image); return file.path; + } catch (const std::exception& e) { s.callbacks.failed(e.what()); } + g_clear_error(&error); g_free(bytes); g_object_unref(image); return {}; } -void Media::record(QString workspace) { - if (audio_) return; - const auto device = QMediaDevices::defaultAudioInput(); - if (device.isNull()) { Q_EMIT failed("No microphone is available in this desktop session"); return; } - format_.setSampleRate(16000); format_.setChannelCount(1); format_.setSampleFormat(QAudioFormat::Int16); - if (!device.isFormatSupported(format_)) format_ = device.preferredFormat(); - try { audioPath_ = destination(workspace, ".wav"); } - catch (const std::exception& e) { Q_EMIT failed(e.what()); return; } - file_ = std::make_unique(audioPath_); - if (!file_->open(QIODevice::WriteOnly)) { Q_EMIT failed(file_->errorString()); file_.reset(); return; } - file_->setPermissions(QFileDevice::ReadOwner | QFileDevice::WriteOwner); - file_->write(wavHeader(format_, 0)); - audio_ = std::make_unique(device, format_); - connect(audio_.get(), &QAudioSource::stateChanged, this, [this](auto state) { - if (state == QAudio::StoppedState && audio_ && audio_->error() != QAudio::NoError) { - Q_EMIT failed("Microphone capture failed"); QTimer::singleShot(0, this, &Media::stopRecording); - } - }); - audio_->start(file_.get()); limit_.start(60000); Q_EMIT recordingState(true); } -void Media::stopRecording() { - if (!audio_) return; - limit_.stop(); audio_->stop(); audio_.reset(); - const auto bytes = file_->size() - 44; - bool good = bytes > 0 && bytes <= 64 * 1024 * 1024 && file_->seek(0); - if (good) good = file_->write(wavHeader(format_, static_cast(bytes))) == 44 && file_->commit(); - if (!good) { file_->cancelWriting(); Q_EMIT failed("Microphone recording was empty or could not be saved"); } - file_.reset(); Q_EMIT recordingState(false); - if (good) Q_EMIT saved(audioPath_); -} -QString Media::snapshot(QString workspace) { - if (!screen_.isActive() || image_.isNull()) { Q_EMIT failed("Share a screen before taking a snapshot"); return {}; } - auto image = image_; QByteArray bytes; - do { - bytes.clear(); QBuffer buffer(&bytes); buffer.open(QIODevice::WriteOnly); - if (!image.save(&buffer, "JPEG", 75)) { Q_EMIT failed("Could not encode the snapshot"); return {}; } - if (bytes.size() <= 650 * 1024) break; - image = image.scaled(image.size() * 0.75, Qt::KeepAspectRatio, Qt::SmoothTransformation); - } while (image.width() >= 160); - if (bytes.size() > 650 * 1024) { Q_EMIT failed("Snapshot is too large"); return {}; } - QString path; - try { path = destination(workspace, ".jpg"); } - catch (const std::exception& e) { Q_EMIT failed(e.what()); return {}; } - QSaveFile file(path); - if (!file.open(QIODevice::WriteOnly)) { Q_EMIT failed(file.errorString()); return {}; } - file.setPermissions(QFileDevice::ReadOwner | QFileDevice::WriteOwner); - if (file.write(bytes) != bytes.size() || !file.commit()) { Q_EMIT failed("Could not save the snapshot"); return {}; } - return path; -} -void Media::stopAll() { shareScreen(false); stopRecording(); } -} --- a/src/smith/desktop/media.hpp +++ b/src/smith/desktop/media.hpp @@ -1,41 +1,27 @@ #pragma once -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include #include - +#include namespace smith::desktop { -class Media : public QObject { - Q_OBJECT +class Media { public: - explicit Media(QObject* parent = nullptr); - ~Media() override; - void shareScreen(bool share); - void record(QString workspace); + struct Callbacks { + std::function frame; + std::function screenState, recordingState; + std::function saved, failed; + }; + explicit Media(Callbacks callbacks); + ~Media(); + void shareScreen(bool share, GtkWindow* parent = nullptr); + void record(const std::string& workspace); void stopRecording(); - QString snapshot(QString workspace); - bool recording() const { return audio_ != nullptr; } + std::string snapshot(const std::string& workspace); + bool recording() const; void stopAll(); -Q_SIGNALS: - void frame(QImage image); - void screenState(bool active); - void recordingState(bool active); - void saved(QString file); - void failed(QString message); + struct State; + friend struct MediaTestAccess; private: - QScreenCapture screen_; - QMediaCaptureSession capture_; - QVideoSink sink_; - QImage image_; - QTimer limit_; - QAudioFormat format_; - QString audioPath_; - std::unique_ptr file_; - std::unique_ptr audio_; + std::shared_ptr state_; }; } --- a/src/smith/desktop/wayfire.cpp +++ b/src/smith/desktop/wayfire.cpp @@ -1,66 +1,113 @@ #include "smith/desktop/wayfire.hpp" -#include -#include -#include -#include -#include +#include +#include +#include +#include namespace smith::desktop { -void Wayfire::request(const QString& method, QJsonObject data, std::function reply) { - const auto endpoint = qEnvironmentVariable("WAYFIRE_SOCKET"); - if (endpoint.isEmpty()) { Q_EMIT status("Wayfire unavailable: WAYFIRE_SOCKET is not set"); return; } - auto* socket = new QLocalSocket(this); - socket->setReadBufferSize(1024 * 1024 + 4); - auto* timer = new QTimer(socket); - timer->setSingleShot(true); - auto done = std::make_shared(false); - auto finish = [socket, timer, done] { - if (*done) return false; - *done = true; timer->stop(); socket->abort(); socket->deleteLater(); return true; +struct Wayfire::State { + Callbacks callbacks; + GCancellable* cancel = g_cancellable_new(); + bool alive = true; + ~State() { g_object_unref(cancel); } +}; +Wayfire::Wayfire(Callbacks callbacks) : state_(std::make_shared()) { + state_->callbacks = std::move(callbacks); +} +Wayfire::~Wayfire() { state_->alive = false; g_cancellable_cancel(state_->cancel); } +namespace { +struct Request { + std::string endpoint, method; + json data; + std::function finish; + GCancellable* cancel = g_cancellable_new(); + guint timer = 0; + ~Request() { if (timer) g_source_remove(timer); g_object_unref(cancel); } +}; +void worker(GTask* task, gpointer, gpointer raw, GCancellable* cancel) { + auto& r = *static_cast(raw); + GError* error = nullptr; + auto* client = g_socket_client_new(); g_socket_client_set_timeout(client, 3); + auto* address = g_unix_socket_address_new(r.endpoint.c_str()); + auto* connection = g_socket_client_connect(client, G_SOCKET_CONNECTABLE(address), cancel, &error); + g_object_unref(address); g_object_unref(client); + if (!connection) { g_task_return_error(task, error); return; } + try { + const auto body = json{{"method", r.method}, {"data", r.data}}.dump(); + guint32 size = GUINT32_TO_LE(static_cast(body.size())); + auto* output = g_io_stream_get_output_stream(G_IO_STREAM(connection)); + auto* input = g_io_stream_get_input_stream(G_IO_STREAM(connection)); + gsize count = 0; + auto check = [&](gboolean ok, gsize expected) { + if (!ok || count != expected) throw std::runtime_error(error ? error->message : "Wayfire closed an incomplete reply"); + }; + auto ok = g_output_stream_write_all(output, &size, 4, &count, cancel, &error); check(ok, 4); + ok = g_output_stream_write_all(output, body.data(), body.size(), &count, cancel, &error); check(ok, body.size()); + ok = g_input_stream_read_all(input, &size, 4, &count, cancel, &error); check(ok, 4); + size = GUINT32_FROM_LE(size); + if (!size || size > 1024 * 1024) throw std::runtime_error("Wayfire sent an oversized reply"); + std::string bytes(size, '\0'); + ok = g_input_stream_read_all(input, bytes.data(), size, &count, cancel, &error); check(ok, size); + auto response = json::parse(bytes); + if (response.is_array() && r.method == "window-rules/list-views") response = json{{"views", response}}; + if (!response.is_object()) throw std::runtime_error("Wayfire sent an invalid reply"); + if (response.contains("error")) throw std::runtime_error(response.at("error").dump()); + g_task_return_pointer(task, new json(std::move(response)), [](gpointer p) { delete static_cast(p); }); + } catch (const std::exception& e) { g_task_return_new_error(task, G_IO_ERROR, G_IO_ERROR_FAILED, "%s", e.what()); } + g_clear_error(&error); g_object_unref(connection); +} +} +void Wayfire::request(std::string method, json data, std::function reply) { + auto state = state_; + const auto* endpoint = g_getenv("WAYFIRE_SOCKET"); + if (!endpoint || !*endpoint) { state->callbacks.status("Wayfire unavailable: WAYFIRE_SOCKET is not set"); return; } + auto* r = new Request; + r->endpoint = endpoint; r->method = std::move(method); r->data = std::move(data); + r->finish = [state, reply](json value, const std::string& error) { + if (!state->alive) return; + if (!error.empty()) state->callbacks.status(error); else reply(std::move(value)); }; - connect(timer, &QTimer::timeout, this, [this, finish] { - if (finish()) Q_EMIT status("Wayfire request timed out"); - }); - connect(socket, &QLocalSocket::errorOccurred, this, [this, socket, finish](auto) { - const auto error = socket->errorString(); - if (finish()) Q_EMIT status("Wayfire: " + error); - }); - connect(socket, &QLocalSocket::connected, socket, [socket, method, data] { - const auto body = QJsonDocument(QJsonObject{{"method", method}, {"data", data}}).toJson(QJsonDocument::Compact); - QByteArray frame(4, '\0'); qToLittleEndian(static_cast(body.size()), frame.data()); - frame += body; socket->write(frame); - }); - auto buffer = std::make_shared(); - connect(socket, &QLocalSocket::readyRead, this, [this, socket, buffer, finish, reply, method] { - buffer->append(socket->readAll()); - if (buffer->size() < 4) return; - const auto size = qFromLittleEndian(buffer->constData()); - if (size == 0 || size > 1024 * 1024 || buffer->size() > 1024 * 1024 + 4) { - if (finish()) Q_EMIT status("Wayfire sent an oversized reply"); - return; - } - if (buffer->size() < size + 4) return; - QJsonParseError error; - auto doc = QJsonDocument::fromJson(buffer->mid(4, size), &error); - if (!finish()) return; - if (error.error != QJsonParseError::NoError || - (!doc.isObject() && !(method == "window-rules/list-views" && doc.isArray()))) { - Q_EMIT status("Wayfire sent an invalid reply"); return; - } - auto object = doc.isArray() ? QJsonObject{{"views", doc.array()}} : doc.object(); - if (object.contains("error")) { Q_EMIT status("Wayfire: " + object.value("error").toString()); return; } - reply(object); - }); - timer->start(3000); socket->connectToServer(endpoint); + // Cancellation from the owner also interrupts blocking, fragmented stream reads. + const auto handler = g_cancellable_connect(state->cancel, G_CALLBACK(+[](GCancellable*, gpointer p) { + g_cancellable_cancel(G_CANCELLABLE(p)); + }), g_object_ref(r->cancel), g_object_unref); + auto* completion = new std::pair, gulong>(state, handler); + auto* task = g_task_new(nullptr, r->cancel, [](GObject*, GAsyncResult* result, gpointer raw) { + std::unique_ptr, gulong>> done(static_cast, gulong>*>(raw)); + auto* request = static_cast(g_task_get_task_data(G_TASK(result))); + g_cancellable_disconnect(done->first->cancel, done->second); + if (request->timer) { g_source_remove(request->timer); request->timer = 0; } + GError* error = nullptr; + std::unique_ptr response(static_cast(g_task_propagate_pointer(G_TASK(result), &error))); + request->finish(response ? *response : json{}, error ? error->message : ""); + g_clear_error(&error); + }, completion); + g_task_set_task_data(task, r, [](gpointer p) { delete static_cast(p); }); + r->timer = g_timeout_add(3000, [](gpointer p) -> gboolean { + auto* request = static_cast(p); request->timer = 0; g_cancellable_cancel(request->cancel); return G_SOURCE_REMOVE; + }, r); + g_task_run_in_thread(task, worker); g_object_unref(task); } void Wayfire::refresh() { - request("window-rules/list-views", {}, [this](const QJsonObject& reply) { - if (!reply.value("views").isArray()) { Q_EMIT status("Wayfire did not return a window list"); return; } - Q_EMIT status("Wayfire connected"); Q_EMIT views(reply.value("views").toArray()); + auto state = state_; + request("window-rules/list-views", json::object(), [state](json reply) { + if (!reply.contains("views") || !reply["views"].is_array()) { state->callbacks.status("Wayfire did not return a window list"); return; } + json views = json::array(); + for (const auto& view : reply["views"]) { + if (!view.is_object() || !view.contains("id") || !view["id"].is_number_integer()) continue; + const auto id = view["id"].get(); + if (id < 0 || id > std::numeric_limits::max()) continue; + auto string = [&](const char* key, const char* fallback) { + return view.contains(key) && view[key].is_string() ? view[key].get() : std::string(fallback); + }; + views.push_back({{"id", id}, {"title", string("title", "Untitled")}, {"app-id", string("app-id", "")}}); + } + state->callbacks.status("Wayfire connected"); state->callbacks.views(views); }); } void Wayfire::focus(int id) { if (id < 0) return; - request("window-rules/focus-view", {{"id", id}}, [this](const QJsonObject&) { Q_EMIT status("Window focused"); }); + auto state = state_; + request("window-rules/focus-view", {{"id", id}}, [state](json) { state->callbacks.status("Window focused"); }); } } --- a/src/smith/desktop/wayfire.hpp +++ b/src/smith/desktop/wayfire.hpp @@ -1,21 +1,23 @@ #pragma once -#include -#include -#include -#include +#include "smith/core/json.hpp" #include +#include +#include namespace smith::desktop { -class Wayfire : public QObject { - Q_OBJECT +class Wayfire { public: - explicit Wayfire(QObject* parent = nullptr) : QObject(parent) {} + struct Callbacks { + std::function views; + std::function status; + }; + explicit Wayfire(Callbacks callbacks); + ~Wayfire(); void refresh(); void focus(int id); -Q_SIGNALS: - void views(QJsonArray views); - void status(QString text); private: - void request(const QString& method, QJsonObject data, std::function reply); + struct State; + std::shared_ptr state_; + void request(std::string method, json data, std::function reply); }; } --- a/src/smith/desktop/window.cpp +++ b/src/smith/desktop/window.cpp @@ -1,194 +1,234 @@ #include "smith/desktop/window.hpp" -#include "smith/desktop/backend.hpp" -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include +#include namespace smith::desktop { -Window::Window(bool preview) : preview_(preview), workspace_(QDir::currentPath()) { - setWindowTitle(preview ? "Smith Desktop · Preview" : "Smith Desktop"); resize(1160, 780); - setMinimumSize(860, 600); - auto* central = new QWidget(this); auto* layout = new QVBoxLayout(central); setCentralWidget(central); - layout->setContentsMargins(16, 16, 16, 16); layout->setSpacing(12); - auto* bar = new QHBoxLayout; - auto* title = new QLabel("Smith"); auto titleFont = title->font(); titleFont.setPointSizeF(titleFont.pointSizeF() * 1.4); - titleFont.setBold(true); title->setFont(titleFont); bar->addWidget(title); - state_ = new QLabel(preview ? "Preview · no active connection" : "Opening local conversation…"); - state_->setAccessibleName("Conversation status"); bar->addWidget(state_, 1); - state_->setTextFormat(Qt::PlainText); - state_->setWordWrap(true); - conversations_ = new QComboBox; conversations_->setAccessibleName("Saved local conversations"); - conversations_->setMinimumContentsLength(14); conversations_->addItem("Saved conversations"); bar->addWidget(conversations_); - new_ = new QPushButton("New conversation"); new_->setEnabled(false); bar->addWidget(new_); layout->addLayout(bar); - auto* workspaceBar = new QHBoxLayout; workspaceBar->addWidget(new QLabel("Workspace")); - directory_ = new QLineEdit(workspace_); directory_->setReadOnly(true); directory_->setAccessibleName("Workspace directory"); - workspaceBar->addWidget(directory_, 1); choose_ = new QPushButton("Choose…"); workspaceBar->addWidget(choose_); - layout->addLayout(workspaceBar); - auto* split = new QSplitter; - auto* chat = new QWidget; auto* chatLayout = new QVBoxLayout(chat); chatLayout->setContentsMargins(0, 0, 8, 0); - transcript_ = new QPlainTextEdit; transcript_->setReadOnly(true); transcript_->setAccessibleName("Conversation"); - chatLayout->addWidget(transcript_, 1); - composer_ = new QPlainTextEdit; composer_->setAccessibleName("Message to Smith"); - composer_->setPlaceholderText("Ask Smith about this workspace. Ctrl+Enter sends."); composer_->setMaximumHeight(128); - chatLayout->addWidget(composer_); - auto* actions = new QHBoxLayout; actions->addWidget(new QLabel("Read-only workspace · capture starts only when you choose"), 1); - cancel_ = new QPushButton("Stop turn"); cancel_->setEnabled(false); actions->addWidget(cancel_); - send_ = new QPushButton("Send"); send_->setDefault(true); send_->setEnabled(false); actions->addWidget(send_); - chatLayout->addLayout(actions); split->addWidget(chat); - auto* side = new QWidget; auto* sideLayout = new QVBoxLayout(side); sideLayout->setContentsMargins(8, 0, 0, 0); - auto* windowsTitle = new QLabel("Desktop windows"); auto headingFont = windowsTitle->font(); headingFont.setBold(true); - windowsTitle->setFont(headingFont); sideLayout->addWidget(windowsTitle); - wayfireState_ = new QLabel("Wayfire is not connected"); wayfireState_->setWordWrap(true); sideLayout->addWidget(wayfireState_); - wayfireState_->setTextFormat(Qt::PlainText); - windows_ = new QListWidget; windows_->setAccessibleName("Wayfire windows"); sideLayout->addWidget(windows_, 1); - auto* windowActions = new QHBoxLayout; auto* refresh = new QPushButton("Refresh"); auto* focus = new QPushButton("Focus"); - windowActions->addWidget(refresh); windowActions->addWidget(focus); sideLayout->addLayout(windowActions); - auto* context = new QPushButton("Use selected window as context"); sideLayout->addWidget(context); - auto* captureTitle = new QLabel("Screen and microphone"); captureTitle->setFont(headingFont); sideLayout->addWidget(captureTitle); - captureState_ = new QLabel("Nothing is being captured"); captureState_->setWordWrap(true); sideLayout->addWidget(captureState_); - captureState_->setTextFormat(Qt::PlainText); - previewImage_ = new QLabel("Choose a screen to preview"); previewImage_->setAlignment(Qt::AlignCenter); - previewImage_->setMinimumSize(260, 160); previewImage_->setMaximumHeight(210); sideLayout->addWidget(previewImage_); - share_ = new QPushButton("Share screen…"); share_->setCheckable(true); sideLayout->addWidget(share_); - snapshot_ = new QPushButton("Attach snapshot to message"); snapshot_->setEnabled(false); sideLayout->addWidget(snapshot_); - microphone_ = new QPushButton("Record microphone"); sideLayout->addWidget(microphone_); - auto* stopSharing = new QPushButton("Stop all capture"); sideLayout->addWidget(stopSharing); - split->addWidget(side); split->setStretchFactor(0, 3); split->setStretchFactor(1, 1); layout->addWidget(split, 1); - auto* shortcut = new QShortcut(QKeySequence("Ctrl+Return"), this); connect(shortcut, &QShortcut::activated, this, &Window::send); - connect(send_, &QPushButton::clicked, this, &Window::send); - connect(new_, &QPushButton::clicked, this, [this] { open(); }); - connect(conversations_, &QComboBox::activated, this, [this](int index) { - const auto session = conversations_->itemData(index).toJsonObject(); - if (session.isEmpty() || busy_) return; - workspace_ = session.value("workspace").toString(); directory_->setText(workspace_); open(session.value("id").toString()); - }); - connect(choose_, &QPushButton::clicked, this, [this] { - const auto folder = QFileDialog::getExistingDirectory(this, "Choose a workspace", workspace_); - if (!folder.isEmpty()) { workspace_ = folder; directory_->setText(folder); open(); } - }); - connect(refresh, &QPushButton::clicked, &wayfire_, &Wayfire::refresh); - connect(focus, &QPushButton::clicked, this, [this] { - if (auto* item = windows_->currentItem()) wayfire_.focus(item->data(Qt::UserRole).toInt()); - }); - connect(context, &QPushButton::clicked, this, [this] { - if (auto* item = windows_->currentItem()) { - const auto metadata = item->data(Qt::UserRole + 1).toJsonObject(); - composer_->appendPlainText("Untrusted desktop window metadata, for reference only:\n" + - QString::fromUtf8(QJsonDocument(metadata).toJson(QJsonDocument::Compact))); - } - }); - connect(&wayfire_, &Wayfire::status, wayfireState_, &QLabel::setText); - connect(&wayfire_, &Wayfire::views, this, [this](const QJsonArray& views) { - const auto selected = windows_->currentItem() ? windows_->currentItem()->data(Qt::UserRole).toInt() : -1; - windows_->clear(); - for (const auto& value : views) { - const auto view = value.toObject(); if (!view.value("id").isDouble()) continue; - auto* item = new QListWidgetItem(view.value("title").toString("Untitled") + "\n" + view.value("app-id").toString()); - item->setData(Qt::UserRole, view.value("id").toInt()); windows_->addItem(item); - item->setData(Qt::UserRole + 1, QJsonObject{{"id", view.value("id")}, {"title", view.value("title")}, - {"app_id", view.value("app-id")}}); - if (item->data(Qt::UserRole).toInt() == selected) windows_->setCurrentItem(item); - } - }); - connect(share_, &QPushButton::toggled, this, [this](bool active) { - share_->setText(active ? "Stop sharing screen" : "Share screen…"); media_.shareScreen(active); - }); - connect(&media_, &Media::screenState, this, [this](bool active) { - QSignalBlocker block(share_); share_->setChecked(active); - share_->setText(active ? "Stop sharing screen" : "Share screen…"); - screenActive_ = active; captureStatus(); - if (!active) snapshot_->setEnabled(false); - }); - connect(&media_, &Media::frame, this, [this](const QImage& image) { - snapshot_->setEnabled(!image.isNull()); - if (image.isNull()) { previewImage_->clear(); previewImage_->setText("Choose a screen to preview"); } - else previewImage_->setPixmap(QPixmap::fromImage(image).scaled(300, 200, Qt::KeepAspectRatio, Qt::SmoothTransformation)); - }); - connect(snapshot_, &QPushButton::clicked, this, [this] { - const auto image = media_.snapshot(workspace_); if (image.isEmpty()) return; - composer_->appendPlainText("Please inspect my screen snapshot with read_image: " + QDir(workspace_).relativeFilePath(image)); - }); - connect(microphone_, &QPushButton::clicked, this, [this] { - if (media_.recording()) media_.stopRecording(); else media_.record(workspace_); - }); - connect(&media_, &Media::recordingState, this, [this](bool active) { - microphone_->setText(active ? "Stop microphone recording" : "Record microphone"); - microphoneActive_ = active; captureStatus(); - }); - connect(&media_, &Media::saved, this, [this](const QString& path) { status("Recording saved: " + QDir(workspace_).relativeFilePath(path)); }); - connect(&media_, &Media::failed, this, &Window::status); - connect(stopSharing, &QPushButton::clicked, &media_, &Media::stopAll); +namespace { +GtkWidget* label(const std::string& text, const char* css = nullptr) { + auto* w = gtk_label_new(text.c_str()); gtk_label_set_xalign(GTK_LABEL(w), 0); gtk_label_set_wrap(GTK_LABEL(w), true); + if (css) gtk_widget_add_css_class(w, css); + return w; +} +GtkWidget* box(GtkOrientation direction, int gap = 10) { return gtk_box_new(direction, gap); } +void add(GtkWidget* box, GtkWidget* child) { gtk_box_append(GTK_BOX(box), child); } +GtkWidget* scroll(GtkWidget* child, bool expand = true) { + auto* w = gtk_scrolled_window_new(); gtk_scrolled_window_set_child(GTK_SCROLLED_WINDOW(w), child); + gtk_scrolled_window_set_policy(GTK_SCROLLED_WINDOW(w), GTK_POLICY_AUTOMATIC, GTK_POLICY_AUTOMATIC); + gtk_widget_set_vexpand(w, expand); return w; +} +std::string text(GtkWidget* view) { + auto* buffer = gtk_text_view_get_buffer(GTK_TEXT_VIEW(view)); GtkTextIter start, end; + gtk_text_buffer_get_bounds(buffer, &start, &end); auto* value = gtk_text_buffer_get_text(buffer, &start, &end, false); + std::string result(value); g_free(value); return result; +} +void appendText(GtkWidget* view, const std::string& value) { + auto* buffer = gtk_text_view_get_buffer(GTK_TEXT_VIEW(view)); GtkTextIter end; gtk_text_buffer_get_end_iter(buffer, &end); + gtk_text_buffer_insert(buffer, &end, value.c_str(), -1); + gtk_text_view_scroll_to_iter(GTK_TEXT_VIEW(view), &end, 0, false, 0, 0); +} +std::string relative(const std::string& workspace, const std::string& path) { + return std::filesystem::path(path).lexically_relative(workspace).string(); +} +struct Dialog { Window* window; std::shared_ptr alive; }; +} +Window::Window(GtkApplication* app, bool preview) : preview_(preview), workspace_(std::filesystem::current_path().string()) { + window_ = gtk_application_window_new(app); g_object_ref(window_); + gtk_window_set_title(widget(), preview ? "Smith Desktop · Preview" : "Smith Desktop"); + gtk_window_set_default_size(widget(), 1240, 900); + gtk_window_set_decorated(widget(), false); + g_signal_connect_after(window_, "map", G_CALLBACK(+[](GtkWidget* window, gpointer) { + auto* surface = gtk_native_get_surface(GTK_NATIVE(window)); + if (GDK_IS_WAYLAND_TOPLEVEL(surface)) + gdk_wayland_toplevel_set_application_id(GDK_TOPLEVEL(surface), "smith-desktop"); + }), nullptr); + gtk_widget_add_css_class(window_, "smith"); layout(); + appearance_ = std::make_unique(gtk_widget_get_display(window_)); + connect(); if (preview_) { append("Smith", "Choose a local workspace, then start a conversation.\n\nYou control which window details and screen snapshots enter your message. Microphone recordings stay in the workspace."); - composer_->setPlaceholderText("Preview mode. No model calls or capture sessions are started."); - for (auto* button : {choose_, refresh, focus, context, share_, microphone_, stopSharing}) button->setEnabled(false); - windows_->addItem("Editor\nexample workspace"); windows_->addItem("Terminal\nlocal session"); - wayfireState_->setText("Preview · live windows appear here"); + for (auto* w : {choose_, refresh_, focus_, context_, share_, microphone_, stopSharing_}) gtk_widget_set_sensitive(w, false); + gtk_list_box_append(GTK_LIST_BOX(windows_), label("Editor · example workspace")); + gtk_list_box_append(GTK_LIST_BOX(windows_), label("Terminal · local session")); + gtk_label_set_text(GTK_LABEL(wayfireState_), "Preview · live windows appear here"); } else { - backend_ = std::make_unique(); - connect(backend_.get(), &Backend::ready, this, [this](const QString&) { ready_ = true; setBusy(busy_); status("Local conversation ready"); }); - connect(backend_.get(), &Backend::running, this, [this] { setBusy(true); }); - connect(backend_.get(), &Backend::sessions, this, [this](const QJsonArray& sessions) { - QSignalBlocker block(conversations_); conversations_->clear(); conversations_->addItem("Saved conversations"); - for (const auto& value : sessions) { - const auto s = value.toObject(); conversations_->addItem(s.value("name").toString() + " · " + s.value("workspace").toString(), s); + Backend::Callbacks callbacks; + callbacks.ready = [this](std::string) { ready_ = true; setBusy(busy_); status("Local conversation ready"); }; + callbacks.running = [this] { setBusy(true); }; + callbacks.line = [this](std::string role, std::string value) { append(role, value); }; + callbacks.phase = [this](std::string value) { status(value); }; + callbacks.failed = [this](std::string error) { append("Error", error); status(error); }; + callbacks.finished = [this] { setBusy(false); if (closing_) close(); }; + callbacks.sessions = [this](json sessions) { + updating_ = true; sessions_ = std::move(sessions); + auto* strings = gtk_string_list_new(nullptr); gtk_string_list_append(strings, "Saved conversations"); + for (const auto& s : sessions_) gtk_string_list_append(strings, (s.value("name", "Conversation") + " · " + s.value("workspace", "")).c_str()); + gtk_drop_down_set_model(GTK_DROP_DOWN(conversations_), G_LIST_MODEL(strings)); g_object_unref(strings); + updating_ = false; + }; + backend_ = std::make_unique(std::move(callbacks)); open(); wayfire_->refresh(); + } +} +Window::~Window() { + *alive_ = false; backend_.reset(); wayfire_.reset(); media_.reset(); appearance_.reset(); + if (!closed_) gtk_window_destroy(widget()); + g_object_unref(window_); +} +void Window::layout() { + auto* root = box(GTK_ORIENTATION_VERTICAL, 12); gtk_widget_add_css_class(root, "content"); gtk_window_set_child(widget(), root); + auto* top = box(GTK_ORIENTATION_HORIZONTAL); add(top, label("Smith", "heading")); + state_ = label(preview_ ? "Preview · no active connection" : "Opening local conversation…", "dim-label"); + gtk_widget_set_hexpand(state_, true); add(top, state_); + const char* items[] = {"Saved conversations", nullptr}; conversations_ = gtk_drop_down_new_from_strings(items); add(top, conversations_); + new_ = gtk_button_new_with_label("New conversation"); gtk_widget_set_sensitive(new_, false); add(top, new_); add(root, top); + auto* workspace = box(GTK_ORIENTATION_HORIZONTAL); add(workspace, label("Workspace", "dim-label")); + directory_ = gtk_entry_new(); gtk_editable_set_text(GTK_EDITABLE(directory_), workspace_.c_str()); + gtk_editable_set_editable(GTK_EDITABLE(directory_), false); gtk_widget_set_hexpand(directory_, true); add(workspace, directory_); + choose_ = gtk_button_new_with_label("Choose…"); add(workspace, choose_); add(root, workspace); + auto* split = gtk_paned_new(GTK_ORIENTATION_HORIZONTAL); gtk_widget_set_vexpand(split, true); add(root, split); + auto* chat = box(GTK_ORIENTATION_VERTICAL); gtk_widget_set_hexpand(chat, true); gtk_paned_set_start_child(GTK_PANED(split), chat); + gtk_paned_set_position(GTK_PANED(split), 800); gtk_paned_set_shrink_start_child(GTK_PANED(split), false); + transcript_ = gtk_text_view_new(); gtk_text_view_set_editable(GTK_TEXT_VIEW(transcript_), false); + gtk_text_view_set_wrap_mode(GTK_TEXT_VIEW(transcript_), GTK_WRAP_WORD_CHAR); add(chat, scroll(transcript_)); + composer_ = gtk_text_view_new(); gtk_text_view_set_wrap_mode(GTK_TEXT_VIEW(composer_), GTK_WRAP_WORD_CHAR); + gtk_text_view_set_accepts_tab(GTK_TEXT_VIEW(composer_), false); + auto* composerScroll = scroll(composer_, false); gtk_widget_set_size_request(composerScroll, -1, 120); add(chat, composerScroll); + add(chat, label("Message to Smith · Ctrl+Enter sends", "dim-label")); + auto* actions = box(GTK_ORIENTATION_HORIZONTAL); auto* note = label("Read-only workspace · capture starts when you choose", "dim-label"); + gtk_widget_set_hexpand(note, true); add(actions, note); + cancel_ = gtk_button_new_with_label("Stop turn"); gtk_widget_set_sensitive(cancel_, false); add(actions, cancel_); + send_ = gtk_button_new_with_label("Send"); gtk_widget_add_css_class(send_, "suggested-action"); gtk_widget_set_sensitive(send_, false); add(actions, send_); add(chat, actions); + auto* side = box(GTK_ORIENTATION_VERTICAL); gtk_widget_add_css_class(side, "inset"); gtk_widget_set_margin_start(side, 12); + gtk_widget_set_size_request(side, 320, -1); gtk_paned_set_end_child(GTK_PANED(split), side); gtk_paned_set_shrink_end_child(GTK_PANED(split), false); + add(side, label("Desktop windows", "section-heading")); wayfireState_ = label("Wayfire is not connected", "dim-label"); add(side, wayfireState_); + windows_ = gtk_list_box_new(); auto* listScroll = scroll(windows_); gtk_widget_set_size_request(listScroll, -1, 140); add(side, listScroll); + auto* windowActions = box(GTK_ORIENTATION_HORIZONTAL); refresh_ = gtk_button_new_with_label("Refresh"); focus_ = gtk_button_new_with_label("Focus"); + add(windowActions, refresh_); add(windowActions, focus_); add(side, windowActions); + context_ = gtk_button_new_with_label("Use selected window as context"); add(side, context_); + add(side, label("Screen and microphone", "section-heading")); captureState_ = label("Nothing is being captured", "dim-label"); add(side, captureState_); + previewImage_ = gtk_picture_new(); gtk_picture_set_content_fit(GTK_PICTURE(previewImage_), GTK_CONTENT_FIT_CONTAIN); + gtk_widget_set_size_request(previewImage_, 280, 160); add(side, previewImage_); + share_ = gtk_toggle_button_new_with_label("Share screen…"); add(side, share_); + snapshot_ = gtk_button_new_with_label("Attach snapshot to message"); gtk_widget_set_sensitive(snapshot_, false); add(side, snapshot_); + microphone_ = gtk_button_new_with_label("Record microphone"); add(side, microphone_); + stopSharing_ = gtk_button_new_with_label("Stop all capture"); add(side, stopSharing_); +} +void Window::connect() { + wayfire_ = std::make_unique(Wayfire::Callbacks{ + [this](json views) { + int selected = -1; + if (auto* row = gtk_list_box_get_selected_row(GTK_LIST_BOX(windows_))) { + if (auto* old = static_cast(g_object_get_data(G_OBJECT(row), "view"))) selected = old->value("id", -1); } - }); - connect(backend_.get(), &Backend::line, this, &Window::append); - connect(backend_.get(), &Backend::phase, this, &Window::status); - connect(backend_.get(), &Backend::failed, this, [this](const QString& error) { append("Error", error); status(error); }); - connect(backend_.get(), &Backend::finished, this, [this] { setBusy(false); if (closing_) close(); }); - connect(cancel_, &QPushButton::clicked, backend_.get(), &Backend::stop); - open(); wayfire_.refresh(); + gtk_list_box_remove_all(GTK_LIST_BOX(windows_)); + for (const auto& view : views) { + if (!view.contains("id") || !view["id"].is_number_integer()) continue; + auto* row = gtk_list_box_row_new(); gtk_list_box_row_set_child(GTK_LIST_BOX_ROW(row), label(view.value("title", "Untitled") + "\n" + view.value("app-id", ""))); + auto* metadata = new json{{"id", view["id"]}, {"title", view.value("title", "")}, {"app_id", view.value("app-id", "")}}; + g_object_set_data_full(G_OBJECT(row), "view", metadata, [](gpointer p) { delete static_cast(p); }); + gtk_list_box_append(GTK_LIST_BOX(windows_), row); + if (view["id"] == selected) gtk_list_box_select_row(GTK_LIST_BOX(windows_), GTK_LIST_BOX_ROW(row)); + } + }, [this](std::string value) { gtk_label_set_text(GTK_LABEL(wayfireState_), value.c_str()); }}); + media_ = std::make_unique(Media::Callbacks{ + [this](GdkTexture* image) { gtk_picture_set_paintable(GTK_PICTURE(previewImage_), image ? GDK_PAINTABLE(image) : nullptr); gtk_widget_set_sensitive(snapshot_, image != nullptr); }, + [this](bool active) { + screenActive_ = active; + g_signal_handlers_block_matched(share_, G_SIGNAL_MATCH_DATA, 0, 0, nullptr, nullptr, this); gtk_toggle_button_set_active(GTK_TOGGLE_BUTTON(share_), active); g_signal_handlers_unblock_matched(share_, G_SIGNAL_MATCH_DATA, 0, 0, nullptr, nullptr, this); + gtk_button_set_label(GTK_BUTTON(share_), active ? "Stop sharing screen" : "Share screen…"); captureStatus(); + }, + [this](bool active) { microphoneActive_ = active; gtk_button_set_label(GTK_BUTTON(microphone_), active ? "Stop microphone recording" : "Record microphone"); captureStatus(); }, + [this](std::string path) { status("Recording saved: " + relative(workspace_, path)); }, + [this](std::string error) { status(error); }}); + auto click = [this](GtkWidget* button, void (*handler)(GtkButton*, gpointer)) { g_signal_connect(button, "clicked", G_CALLBACK(handler), this); }; + click(send_, [](GtkButton*, gpointer p) { static_cast(p)->send(); }); + click(new_, [](GtkButton*, gpointer p) { static_cast(p)->open(); }); + click(cancel_, [](GtkButton*, gpointer p) { auto* self = static_cast(p); if (self->backend_) self->backend_->stop(); }); + click(refresh_, [](GtkButton*, gpointer p) { static_cast(p)->wayfire_->refresh(); }); + click(focus_, [](GtkButton*, gpointer p) { + auto* self = static_cast(p); + if (auto* row = gtk_list_box_get_selected_row(GTK_LIST_BOX(self->windows_))) + if (auto* view = static_cast(g_object_get_data(G_OBJECT(row), "view"))) self->wayfire_->focus(view->value("id", -1)); + }); + click(context_, [](GtkButton*, gpointer p) { + auto* self = static_cast(p); + if (auto* row = gtk_list_box_get_selected_row(GTK_LIST_BOX(self->windows_))) + if (auto* view = static_cast(g_object_get_data(G_OBJECT(row), "view"))) + appendText(self->composer_, "\nUntrusted desktop window metadata, for reference only:\n" + view->dump()); + }); + click(snapshot_, [](GtkButton*, gpointer p) { + auto* self = static_cast(p); auto path = self->media_->snapshot(self->workspace_); + if (!path.empty()) appendText(self->composer_, "\nPlease inspect my screen snapshot with read_image: " + relative(self->workspace_, path)); + }); + click(microphone_, [](GtkButton*, gpointer p) { + auto* self = static_cast(p); + if (self->media_->recording()) self->media_->stopRecording(); else self->media_->record(self->workspace_); + }); + click(stopSharing_, [](GtkButton*, gpointer p) { static_cast(p)->media_->stopAll(); }); + g_signal_connect(share_, "toggled", G_CALLBACK(+[](GtkToggleButton* button, gpointer p) { + auto* self = static_cast(p); self->media_->shareScreen(gtk_toggle_button_get_active(button), self->widget()); + }), this); + g_signal_connect(conversations_, "notify::selected", G_CALLBACK(+[](GObject* object, GParamSpec*, gpointer p) { + auto* self = static_cast(p); const auto index = gtk_drop_down_get_selected(GTK_DROP_DOWN(object)); + if (self->updating_ || self->busy_ || !index || index > self->sessions_.size()) return; + const auto session = self->sessions_[index - 1]; self->workspace_ = session.value("workspace", ""); + gtk_editable_set_text(GTK_EDITABLE(self->directory_), self->workspace_.c_str()); self->open(session.value("id", "")); + }), this); + click(choose_, [](GtkButton*, gpointer p) { + auto* self = static_cast(p); auto* dialog = gtk_file_dialog_new(); gtk_file_dialog_set_title(dialog, "Choose a workspace"); + auto* initial = g_file_new_for_path(self->workspace_.c_str()); gtk_file_dialog_set_initial_folder(dialog, initial); g_object_unref(initial); + gtk_file_dialog_select_folder(dialog, self->widget(), nullptr, [](GObject* source, GAsyncResult* result, gpointer raw) { + std::unique_ptr data(static_cast(raw)); GError* error = nullptr; + auto* folder = gtk_file_dialog_select_folder_finish(GTK_FILE_DIALOG(source), result, &error); + if (folder && *data->alive) { + auto* path = g_file_get_path(folder); + if (path) { auto* self = data->window; self->workspace_ = path; gtk_editable_set_text(GTK_EDITABLE(self->directory_), path); self->open(); g_free(path); } + } + g_clear_object(&folder); g_clear_error(&error); + }, new Dialog{self, self->alive_}); g_object_unref(dialog); + }); + auto* keys = gtk_event_controller_key_new(); gtk_widget_add_controller(composer_, keys); + g_signal_connect(keys, "key-pressed", G_CALLBACK(+[](GtkEventControllerKey*, guint key, guint, GdkModifierType modifiers, gpointer p) -> gboolean { + if ((key == GDK_KEY_Return || key == GDK_KEY_KP_Enter) && (modifiers & GDK_CONTROL_MASK)) { static_cast(p)->send(); return true; } + return false; + }), this); + g_signal_connect(window_, "close-request", G_CALLBACK(+[](GtkWindow*, gpointer p) -> gboolean { static_cast(p)->closeRequest(); return true; }), this); +} +void Window::captureStatus() { + std::string value; + if (screenActive_) value = "Screen preview active · attach a snapshot to share"; + if (microphoneActive_) value += (value.empty() ? "" : "\n") + std::string("Microphone recording · 60 second limit"); + gtk_label_set_text(GTK_LABEL(captureState_), value.empty() ? "Nothing is being captured" : value.c_str()); +} +void Window::status(const std::string& value) { gtk_label_set_text(GTK_LABEL(state_), value.c_str()); } +void Window::append(const std::string& role, const std::string& value) { appendText(transcript_, role + "\n" + value + "\n\n"); } +void Window::setBusy(bool busy) { + busy_ = busy; gtk_widget_set_sensitive(send_, ready_ && !busy); gtk_widget_set_sensitive(cancel_, busy && ready_); + for (auto* w : {choose_, new_, conversations_}) gtk_widget_set_sensitive(w, !busy && !preview_); +} +void Window::open(std::string session) { + if (busy_ || !backend_) return; + media_->stopAll(); gtk_text_buffer_set_text(gtk_text_view_get_buffer(GTK_TEXT_VIEW(composer_)), "", 0); + ready_ = false; setBusy(true); gtk_text_buffer_set_text(gtk_text_view_get_buffer(GTK_TEXT_VIEW(transcript_)), "", 0); + status("Opening local conversation…"); backend_->open(workspace_, std::move(session)); +} +void Window::send() { + auto value = text(composer_); if (!ready_ || busy_ || value.find_first_not_of(" \t\r\n") == std::string::npos) return; + append("You", value); gtk_text_buffer_set_text(gtk_text_view_get_buffer(GTK_TEXT_VIEW(composer_)), "", 0); setBusy(true); backend_->send(std::move(value)); +} +void Window::close() { media_->stopAll(); backend_.reset(); wayfire_.reset(); closed_ = true; *alive_ = false; gtk_window_destroy(widget()); } +void Window::closeRequest() { + media_->stopAll(); + if (busy_ && ready_) { + if (closing_) return; + auto* dialog = gtk_alert_dialog_new("Stop Smith's active turn before closing?"); + const char* buttons[] = {"Keep open", "Stop and close", nullptr}; gtk_alert_dialog_set_buttons(dialog, buttons); + gtk_alert_dialog_set_cancel_button(dialog, 0); gtk_alert_dialog_set_default_button(dialog, 0); + gtk_alert_dialog_choose(dialog, widget(), nullptr, [](GObject* source, GAsyncResult* result, gpointer raw) { + std::unique_ptr data(static_cast(raw)); GError* error = nullptr; + const auto choice = gtk_alert_dialog_choose_finish(GTK_ALERT_DIALOG(source), result, &error); g_clear_error(&error); + if (*data->alive && choice == 1) { auto* self = data->window; self->closing_ = true; self->backend_->stop(); self->status("Stopping active turn before closing…"); } + }, new Dialog{this, alive_}); g_object_unref(dialog); return; } -} -Window::~Window() { media_.stopAll(); } -void Window::captureStatus() { - QStringList state; - if (screenActive_) state << "Screen preview active · attach a snapshot to share"; - if (microphoneActive_) state << "Microphone recording · 60 second limit"; - captureState_->setText(state.isEmpty() ? "Nothing is being captured" : state.join('\n')); -} -void Window::status(QString text) { state_->setText(text); } -void Window::append(QString role, QString text) { transcript_->appendPlainText(role + "\n" + text + "\n"); } -void Window::setBusy(bool busy) { - busy_ = busy; send_->setEnabled(ready_ && !busy); cancel_->setEnabled(busy && ready_); - choose_->setEnabled(!busy && !preview_); new_->setEnabled(!busy && !preview_); conversations_->setEnabled(!busy && !preview_); -} -void Window::open(QString session) { - if (busy_ || !backend_) return; - media_.stopAll(); composer_->clear(); - ready_ = false; setBusy(true); transcript_->clear(); status("Opening local conversation…"); backend_->open(workspace_, session); -} -void Window::send() { - const auto message = composer_->toPlainText().trimmed(); if (!ready_ || busy_ || message.isEmpty()) return; - append("You", message); composer_->clear(); setBusy(true); backend_->send(message); -} -void Window::closeEvent(QCloseEvent* event) { - media_.stopAll(); - if (busy_ && ready_ && !closing_) { - if (QMessageBox::question(this, "Stop the active turn?", "Stop Smith's active turn before closing?") != QMessageBox::Yes) { - event->ignore(); return; - } - closing_ = true; backend_->stop(); status("Stopping active turn before closing…"); event->ignore(); return; - } - if (busy_ && ready_) { event->ignore(); return; } - event->accept(); -} -} + close(); +} +} --- a/src/smith/desktop/window.hpp +++ b/src/smith/desktop/window.hpp @@ -1,43 +1,39 @@ #pragma once -#include -#include +#include +#include "smith/core/json.hpp" +#include "smith/desktop/backend.hpp" +#include "smith/desktop/wayfire.hpp" #include "smith/desktop/media.hpp" -#include "smith/desktop/wayfire.hpp" +#include "smith/desktop/appearance.hpp" -class QLabel; -class QLineEdit; -class QListWidget; -class QPlainTextEdit; -class QPushButton; -class QCloseEvent; -class QComboBox; namespace smith::desktop { -class Backend; -class Window : public QMainWindow { - Q_OBJECT +class Window { public: - explicit Window(bool preview = false); - ~Window() override; -protected: - void closeEvent(QCloseEvent* event) override; + Window(GtkApplication* app, bool preview = false); + ~Window(); + GtkWindow* widget() const { return GTK_WINDOW(window_); } private: - void open(QString session = {}); + void layout(); + void connect(); + void open(std::string session = {}); void send(); - void append(QString role, QString text); - void status(QString text); + void append(const std::string& role, const std::string& text); + void status(const std::string& text); void setBusy(bool busy); void captureStatus(); - bool preview_, busy_ = false, ready_ = false, closing_ = false; + void close(); + void closeRequest(); + bool preview_, busy_ = false, ready_ = false, closing_ = false, closed_ = false, updating_ = false; bool screenActive_ = false, microphoneActive_ = false; - QString workspace_; + std::string workspace_; + json sessions_ = json::array(); + GtkWidget *window_, *directory_, *conversations_, *transcript_, *composer_, *windows_; + GtkWidget *state_, *wayfireState_, *previewImage_, *captureState_; + GtkWidget *send_, *cancel_, *new_, *choose_, *share_, *snapshot_, *microphone_, *refresh_, *focus_, *context_, *stopSharing_; std::unique_ptr backend_; - Wayfire wayfire_; - Media media_; - QLineEdit* directory_; - QComboBox* conversations_; - QPlainTextEdit *transcript_, *composer_; - QListWidget* windows_; - QLabel *state_, *wayfireState_, *previewImage_, *captureState_; - QPushButton *send_, *cancel_, *new_, *choose_, *share_, *snapshot_, *microphone_; + std::unique_ptr wayfire_; + std::unique_ptr media_; + std::unique_ptr appearance_; + std::shared_ptr alive_ = std::make_shared(true); }; } --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -8,12 +8,11 @@ set_tests_properties(unit PROPERTIES TIMEOUT 60) if(SMITH_BUILD_DESKTOP) add_executable(smith-desktop-test desktop_test.cpp) - set_target_properties(smith-desktop-test PROPERTIES AUTOMOC ON) - target_link_libraries(smith-desktop-test PRIVATE smith_desktop Qt6::Test) + target_link_libraries(smith-desktop-test PRIVATE smith_desktop Catch2::Catch2WithMain) add_test(NAME desktop-ipc COMMAND smith-desktop-test) - set_tests_properties(desktop-ipc PROPERTIES TIMEOUT 15) + set_tests_properties(desktop-ipc PROPERTIES TIMEOUT 20) add_test(NAME desktop-preview COMMAND smith-desktop --smoke-test) - set_tests_properties(desktop-preview PROPERTIES ENVIRONMENT "QT_QPA_PLATFORM=offscreen" TIMEOUT 15) + set_tests_properties(desktop-preview PROPERTIES ENVIRONMENT "GSK_RENDERER=cairo" TIMEOUT 15) endif() add_test(NAME perf COMMAND smith-unit "[perf]") set_tests_properties(perf PROPERTIES TIMEOUT 300) --- a/tests/desktop_test.cpp +++ b/tests/desktop_test.cpp @@ -1,95 +1,175 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include +#include #include "smith/desktop/wayfire.hpp" #include "smith/desktop/backend.hpp" +#include "smith/desktop/capture-files.hpp" +#include "smith/desktop/media-internal.hpp" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include -class DesktopTest : public QObject { - Q_OBJECT -private Q_SLOTS: - void overlappingOpenCannotFinishTheOwningOperation() { - QTemporaryDir dir; - QLocalServer bootstrap; - QVERIFY(bootstrap.listen(dir.filePath("bootstrap.sock"))); - QDir().mkpath(dir.filePath("smith")); - QFile config(dir.filePath("smith/smith.toml")); QVERIFY(config.open(QIODevice::WriteOnly)); - config.write(("bootstrap_socket = \"" + bootstrap.fullServerName() + "\"\nstate_directory = \"" + - dir.filePath("state") + "\"\n").toUtf8()); config.close(); - auto old = qgetenv("XDG_CONFIG_HOME"); qputenv("XDG_CONFIG_HOME", dir.path().toUtf8()); - { - smith::desktop::Backend backend; - QSignalSpy failed(&backend, &smith::desktop::Backend::failed); - QSignalSpy finished(&backend, &smith::desktop::Backend::finished); - backend.open(dir.path()); - QVERIFY(bootstrap.waitForNewConnection(2000)); // deliberately withhold a bootstrap reply - backend.open(dir.path()); - QVERIFY(failed.wait(2000)); - QVERIFY(failed.first().first().toString().contains("running")); - QCOMPARE(finished.size(), 0); +using namespace smith::desktop; +namespace { +struct Temp { + std::string path; + Temp() { auto* p = g_dir_make_tmp("smith-gtk4-test-XXXXXX", nullptr); REQUIRE(p); path = p; g_free(p); } + ~Temp() { std::filesystem::remove_all(path); } +}; +struct Env { + std::string name; std::optional previous; + Env(const char* key, const std::string& value) : name(key) { + if (const auto* p = g_getenv(key)) previous = p; + g_setenv(key, value.c_str(), true); + } + ~Env() { if (previous) g_setenv(name.c_str(), previous->c_str(), true); else g_unsetenv(name.c_str()); } +}; +bool wait(const std::function& done) { + const auto end = std::chrono::steady_clock::now() + std::chrono::seconds(3); + while (!done() && std::chrono::steady_clock::now() < end) { + while (g_main_context_iteration(nullptr, false)) {} + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + } + return done(); +} +bool writeAll(int fd, const void* data, std::size_t size) { + const auto* bytes = static_cast(data); + while (size) { + const auto count = ::write(fd, bytes, size); + if (count < 0 && errno == EINTR) continue; + if (count <= 0) return false; + bytes += count; size -= static_cast(count); + } + return true; +} +std::string receive(int fd) { + guint32 size; if (::recv(fd, &size, 4, MSG_WAITALL) != 4) return {}; + size = GUINT32_FROM_LE(size); if (size > 1024 * 1024) return {}; + std::string bytes(size, '\0'); + return ::recv(fd, bytes.data(), size, MSG_WAITALL) == size ? bytes : std::string{}; +} +struct Server { + int fd; + std::string path; + explicit Server(const std::string& directory) : path(directory + "/wayfire.sock") { + fd = ::socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0); REQUIRE(fd >= 0); + sockaddr_un addr{}; addr.sun_family = AF_UNIX; std::strcpy(addr.sun_path, path.c_str()); + REQUIRE(::bind(fd, reinterpret_cast(&addr), sizeof(addr)) == 0); REQUIRE(::listen(fd, 4) == 0); + } + ~Server() { ::close(fd); } +}; +} +TEST_CASE("Wayfire accepts a fragmented top-level window list") { + Temp temp; Server server(temp.path); Env endpoint("WAYFIRE_SOCKET", server.path); + std::jthread thread([&] { + int client = ::accept(server.fd, nullptr, nullptr); if (client < 0) return; + const auto request = receive(client); + if (request.find("window-rules/list-views") != std::string::npos) { + const std::string body = R"([{"id":42,"title":"Editor"}])"; + guint32 length = GUINT32_TO_LE(body.size()); + if (writeAll(client, &length, 2)) { + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + if (writeAll(client, reinterpret_cast(&length) + 2, 2)) { + const bool sent = writeAll(client, body.data(), body.size()); (void)sent; + } + } } - if (old.isNull()) qunsetenv("XDG_CONFIG_HOME"); else qputenv("XDG_CONFIG_HOME", old); + ::close(client); + }); + smith::json views; std::string status; + Wayfire wayfire({[&](smith::json value) { views = value; }, [&](std::string value) { status = value; }}); + wayfire.refresh(); REQUIRE(wait([&] { return views.is_array() || (!status.empty() && status != "Wayfire connected"); })); + REQUIRE(views.is_array()); REQUIRE(views[0]["id"] == 42); +} +TEST_CASE("Wayfire rejects oversized frames before allocating their body") { + Temp temp; Server server(temp.path); Env endpoint("WAYFIRE_SOCKET", server.path); + std::jthread thread([&] { + int client = ::accept(server.fd, nullptr, nullptr); if (client < 0) return; + if (!receive(client).empty()) { + guint32 length = GUINT32_TO_LE(2 * 1024 * 1024); const bool sent = writeAll(client, &length, 4); (void)sent; + } + ::close(client); + }); + std::string status; bool views = false; + Wayfire wayfire({[&](smith::json) { views = true; }, [&](std::string value) { status = value; }}); + wayfire.refresh(); REQUIRE(wait([&] { return !status.empty(); })); REQUIRE_FALSE(views); REQUIRE(status.find("oversized") != std::string::npos); +} +TEST_CASE("Missing desktop session does not probe other hosts") { + Env endpoint("WAYFIRE_SOCKET", ""); std::string status; + Wayfire wayfire({[](smith::json) {}, [&](std::string value) { status = value; }}); + wayfire.refresh(); REQUIRE(status.find("not set") != std::string::npos); +} +TEST_CASE("Missing local bootstrap fails before login") { + Temp temp; Env config("XDG_CONFIG_HOME", temp.path); std::string failure; + Backend::Callbacks callbacks; callbacks.failed = [&](std::string value) { failure = value; }; + Backend backend(std::move(callbacks)); backend.open(temp.path); + REQUIRE(wait([&] { return !failure.empty(); })); REQUIRE(failure.find("bootstrap_socket") != std::string::npos); +} +TEST_CASE("Captures are private and symlink directories are refused") { + Temp temp; + { + CaptureFile capture(temp.path, ".jpg"); const auto name = capture.path; + REQUIRE(::write(capture.fd, "image", 5) == 5); capture.commit(); + struct stat st{}; REQUIRE(::stat(name.c_str(), &st) == 0); REQUIRE((st.st_mode & 0777) == 0600); + REQUIRE(std::filesystem::file_size(name) == 5); } - void absentLocalBootstrapFailsBeforeLogin() { - QTemporaryDir config; - auto old = qgetenv("XDG_CONFIG_HOME"); qputenv("XDG_CONFIG_HOME", config.path().toUtf8()); - smith::desktop::Backend backend; - QSignalSpy failed(&backend, &smith::desktop::Backend::failed); - backend.open(config.path()); QVERIFY(failed.wait(2000)); - QVERIFY(failed.first().first().toString().contains("bootstrap_socket")); - if (old.isNull()) qunsetenv("XDG_CONFIG_HOME"); else qputenv("XDG_CONFIG_HOME", old); - } - void fragmentedWindowList() { - QTemporaryDir dir; - QLocalServer server; - QVERIFY(server.listen(dir.filePath("wayfire.sock"))); - qputenv("WAYFIRE_SOCKET", server.fullServerName().toUtf8()); - connect(&server, &QLocalServer::newConnection, &server, [&] { - auto* client = server.nextPendingConnection(); - connect(client, &QLocalSocket::readyRead, client, [client] { - const auto request = client->readAll(); - QVERIFY(request.contains("window-rules/list-views")); - // The installed Wayfire window-rules plugin returns a top-level array. - const auto bytes = QJsonDocument(QJsonArray{QJsonObject{{"id", 42}, {"title", "Editor"}}}).toJson(); - QByteArray frame(4, '\0'); qToLittleEndian(bytes.size(), frame.data()); frame += bytes; - client->write(frame.first(2)); - QTimer::singleShot(0, client, [client, frame] { client->write(frame.mid(2)); }); - }); - }); - smith::desktop::Wayfire wayfire; - QSignalSpy views(&wayfire, &smith::desktop::Wayfire::views); - wayfire.refresh(); QVERIFY(views.wait(2000)); - const auto result = qvariant_cast(views.first().first()); - QCOMPARE(result.first().toObject().value("id").toInt(), 42); - } - void oversizedReplyFailsClosed() { - QTemporaryDir dir; QLocalServer server; QVERIFY(server.listen(dir.filePath("ipc.sock"))); - qputenv("WAYFIRE_SOCKET", server.fullServerName().toUtf8()); - connect(&server, &QLocalServer::newConnection, &server, [&] { - auto* client = server.nextPendingConnection(); - connect(client, &QLocalSocket::readyRead, client, [client] { - client->readAll(); QByteArray frame(4, '\0'); - qToLittleEndian(2 * 1024 * 1024, frame.data()); client->write(frame); - }); - }); - smith::desktop::Wayfire wayfire; QSignalSpy status(&wayfire, &smith::desktop::Wayfire::status); - QSignalSpy views(&wayfire, &smith::desktop::Wayfire::views); - wayfire.refresh(); QVERIFY(status.wait(2000)); QCOMPARE(views.size(), 0); - QVERIFY(status.first().first().toString().contains("oversized")); - } - void missingSessionDoesNotProbeOtherHosts() { - qunsetenv("WAYFIRE_SOCKET"); smith::desktop::Wayfire wayfire; - QSignalSpy status(&wayfire, &smith::desktop::Wayfire::status); - wayfire.refresh(); QCOMPARE(status.size(), 1); - QVERIFY(status.first().first().toString().contains("not set")); - } + Temp unsafe; std::filesystem::create_directory_symlink(temp.path, unsafe.path + "/.smith"); + REQUIRE_THROWS(CaptureFile(unsafe.path, ".jpg")); +} + +TEST_CASE("Overlapping open cannot finish the owning operation") { + Temp temp; Server bootstrap(temp.path); Env config("XDG_CONFIG_HOME", temp.path); + std::filesystem::create_directories(temp.path + "/smith"); + std::ofstream settings(temp.path + "/smith/smith.toml"); + settings << "bootstrap_socket = \"" << bootstrap.path << "\"\nstate_directory = \"" << temp.path << "/state\"\n"; settings.close(); + std::string failure; int finished = 0; + Backend::Callbacks callbacks; callbacks.failed = [&](std::string value) { failure = value; }; callbacks.finished = [&] { ++finished; }; + Backend backend(std::move(callbacks)); backend.open(temp.path); + REQUIRE(wait([&] { pollfd p{bootstrap.fd, POLLIN, 0}; return ::poll(&p, 1, 0) > 0; })); + const int client = ::accept(bootstrap.fd, nullptr, nullptr); REQUIRE(client >= 0); + backend.open(temp.path); REQUIRE(wait([&] { return !failure.empty(); })); + REQUIRE(failure.find("running") != std::string::npos); REQUIRE(finished == 0); + ::close(client); +} + +namespace smith::desktop { +struct MediaTestAccess { + static Media::State& state(Media& media) { return *media.state_; } }; -QTEST_GUILESS_MAIN(DesktopTest) -#include "desktop_test.moc" +} +TEST_CASE("Synthetic video frames become bounded private JPEG snapshots") { + gst_init(nullptr, nullptr); Temp temp; int frames = 0; + Media media({[&](GdkTexture* texture) { + if (texture) { ++frames; REQUIRE(gdk_texture_get_width(texture) == 64); REQUIRE(gdk_texture_get_height(texture) == 48); } + }, [](bool) {}, [](bool) {}, [](std::string) {}, [](std::string) {}}); + auto& state = MediaTestAccess::state(media); + state.screen = gst_parse_launch("videotestsrc num-buffers=1 ! video/x-raw,format=RGBA,width=64,height=48 ! appsink name=preview sync=false", nullptr); + REQUIRE(state.screen); REQUIRE(gst_element_set_state(state.screen, GST_STATE_PLAYING) != GST_STATE_CHANGE_FAILURE); + REQUIRE(wait([&] { poll_frame(&state); return frames > 0; })); + const auto path = media.snapshot(temp.path); REQUIRE_FALSE(path.empty()); + REQUIRE(std::filesystem::file_size(path) < 650 * 1024); + GError* error = nullptr; auto* image = gdk_pixbuf_new_from_file(path.c_str(), &error); + REQUIRE(error == nullptr); REQUIRE(image); REQUIRE(gdk_pixbuf_get_width(image) == 64); g_object_unref(image); + media.stopAll(); REQUIRE(state.image == nullptr); REQUIRE(state.screen == nullptr); + REQUIRE(media.snapshot(temp.path).empty()); +} +TEST_CASE("Synthetic microphone stream is finalized as a private WAV") { + gst_init(nullptr, nullptr); Temp temp; std::string saved, error; + Media media({[](GdkTexture*) {}, [](bool) {}, [](bool) {}, [&](std::string value) { saved = value; }, [&](std::string value) { error = value; }}); + auto& state = MediaTestAccess::state(media); state.audioFile = std::make_unique(temp.path, ".wav"); + state.audio = gst_parse_launch("audiotestsrc is-live=true ! audioconvert ! audio/x-raw,format=S16LE,rate=16000,channels=1 ! wavenc ! fdsink name=recording sync=false", nullptr); + REQUIRE(state.audio); auto* sink = gst_bin_get_by_name(GST_BIN(state.audio), "recording"); + g_object_set(sink, "fd", state.audioFile->fd, nullptr); gst_object_unref(sink); + REQUIRE(gst_element_set_state(state.audio, GST_STATE_PLAYING) != GST_STATE_CHANGE_FAILURE); + REQUIRE(wait([&] { struct stat st{}; return ::fstat(state.audioFile->fd, &st) == 0 && st.st_size > 100; })); + media.stopRecording(); REQUIRE(error.empty()); REQUIRE_FALSE(saved.empty()); REQUIRE_FALSE(media.recording()); + std::ifstream input(saved, std::ios::binary); char header[44]; input.read(header, 44); + REQUIRE(std::string(header, 4) == "RIFF"); REQUIRE(std::string(header + 8, 4) == "WAVE"); + guint32 size; std::memcpy(&size, header + 4, 4); REQUIRE(GUINT32_FROM_LE(size) + 8 == std::filesystem::file_size(saved)); +}