Release v0.3.2

- update virtual key list URL
- remove references to xmodmap
- ensure layouts are consistent with length
- remove alt_layout for now

.github/workflows/make-release.yml

@@ -0,0 +1,62 @@
+name: Make Release
+
+on:
+  push:
+    tags:
+      - 'v**'
+
+env:
+  APPDIR: WlxOverlay-S.AppDir
+  
+jobs:
+  make_release:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v3
+      
+      - name: Prepare Environment
+        run: |
+          sudo add-apt-repository -syn universe
+          sudo add-apt-repository -syn ppa:pipewire-debian/pipewire-upstream || sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 25088A0359807596
+          sudo apt-get update
+          sudo apt-get install fuse cmake pkg-config fontconfig libasound2-dev libxkbcommon-dev libopenxr-dev libfontconfig-dev libdbus-1-dev libpipewire-0.3-0 libpipewire-0.3-dev libspa-0.2-dev libx11-6 libxext6 libxrandr2 libx11-dev libxext-dev libxrandr-dev libopenvr-dev libopenvr-api1
+          rustup update
+          
+          test -f linuxdeploy-x86_64.AppImage || wget -q "https://github.com/linuxdeploy/linuxdeploy/releases/download/continuous/linuxdeploy-x86_64.AppImage"
+          chmod +x linuxdeploy-x86_64.AppImage
+          
+          test -d ${APPDIR} && rm -rf ${APPDIR}
+          mkdir -p ${APPDIR}/usr/bin 
+          
+      - name: Cargo Build
+        run: |
+          cargo build --release
+          mv target/release/wlx-overlay-s ${APPDIR}/usr/bin
+          chmod +x ${APPDIR}/usr/bin/wlx-overlay-s
+
+      - name: Package AppImage
+        run: |
+          export VERSION=$GITHUB_REF_NAME
+          ./linuxdeploy-x86_64.AppImage -dwlx-overlay-s.desktop -iwlx-overlay-s.png --appdir=${APPDIR} --output appimage --exclude-library '*libpipewire*'
+          mv WlxOverlay-S-$VERSION-x86_64.AppImage WlxOverlay-S-x86_64.AppImage
+
+      - name: Create Release
+        id: create_release
+        uses: actions/create-release@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.RELEASE_KEY }}
+        with:
+          tag_name: ${{ github.ref_name }}
+          release_name: WlxOverlay-S ${{ github.ref_name }}
+          draft: true
+          prerelease: false
+
+      - name: Upload AppImage
+        uses: actions/upload-release-asset@v1
+        env:
+          GITHUB_TOKEN: ${{ secrets.RELEASE_KEY }}
+        with:
+          upload_url: ${{ steps.create_release.outputs.upload_url }}
+          asset_path: ./WlxOverlay-S-x86_64.AppImage
+          asset_name: WlxOverlay-S-${{ github.ref_name }}-x86_64.AppImage
+          asset_content_type: application/octet-stream

.gitignore

@@ -0,0 +1,2 @@
+/target
+.gdb_history

Cargo.toml

@@ -0,0 +1,56 @@
+[profile.release-with-debug]
+inherits = "release"
+debug = true
+
+[package]
+name = "wlx-overlay-s"
+version = "0.3.2"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow = "1.0.79"
+ash = "^0.37.2"
+chrono = "0.4.29"
+chrono-tz = "0.8.5"
+clap = { version = "4.5.1", features = ["derive"] }
+cstr = "0.2.11"
+ctrlc = { version = "3.4.2", features = ["termination"] }
+dbus = { version = "0.9.7" }
+flexi_logger = "0.27.4"
+fontconfig-rs = "0.1.1"
+freetype-rs = "0.32.0"
+futures = "0.3.29"
+glam = { version = "0.24.1", features = ["approx"] }
+idmap = "0.2.21"
+idmap-derive = "0.1.2"
+input-linux = "0.6.0"
+json = { version = "0.12.4", optional = true }
+libc = "0.2.153"
+log = "0.4.21"
+once_cell = "1.19.0"
+openxr = { version = "0.17.1", features = ["linked"], optional = true }
+ovr_overlay = { features = ["ovr_input", "ovr_system"], git = "https://github.com/galister/ovr_overlay_oyasumi", optional = true }
+regex = "1.9.5"
+rodio = { version = "0.17.1", default-features = false, features = ["wav", "hound"] }
+rosc = { version = "0.10.1", optional = true }
+serde = { version = "1.0.188", features = ["derive", "rc"] }
+serde_json = "1.0.113"
+serde_yaml = "0.9.25"
+smallvec = "1.11.0"
+strum = { version = "0.25.0", features = ["derive"] }
+thiserror = "1.0.56"
+vulkano = { git = "https://github.com/vulkano-rs/vulkano", rev = "94f50f1" }
+vulkano-shaders = { git = "https://github.com/vulkano-rs/vulkano", rev = "94f50f1" }
+wlx-capture = { git = "https://github.com/galister/wlx-capture", tag = "v0.3.1", default-features = false }
+xdg = "2.5.2"
+
+[features]
+default = ["openvr", "openxr", "osc", "x11", "wayland"] 
+openvr = ["dep:ovr_overlay", "dep:json"]
+openxr = ["dep:openxr"]
+osc = ["dep:rosc"]
+x11 = ["wlx-capture/xshm"]
+wayland = ["wlx-capture/pipewire", "wlx-capture/wlr"]
+

LICENSE

@@ -0,0 +1,674 @@
+                    GNU GENERAL PUBLIC LICENSE
+                       Version 3, 29 June 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+                            Preamble
+
+  The GNU General Public License is a free, copyleft license for
+software and other kinds of works.
+
+  The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works.  By contrast,
+the GNU General Public License is intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.  We, the Free Software Foundation, use the
+GNU General Public License for most of our software; it applies also to
+any other work released this way by its authors.  You can apply it to
+your programs, too.
+
+  When we speak of free software, we are referring to freedom, not
+price.  Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+  To protect your rights, we need to prevent others from denying you
+these rights or asking you to surrender the rights.  Therefore, you have
+certain responsibilities if you distribute copies of the software, or if
+you modify it: responsibilities to respect the freedom of others.
+
+  For example, if you distribute copies of such a program, whether
+gratis or for a fee, you must pass on to the recipients the same
+freedoms that you received.  You must make sure that they, too, receive
+or can get the source code.  And you must show them these terms so they
+know their rights.
+
+  Developers that use the GNU GPL protect your rights with two steps:
+(1) assert copyright on the software, and (2) offer you this License
+giving you legal permission to copy, distribute and/or modify it.
+
+  For the developers' and authors' protection, the GPL clearly explains
+that there is no warranty for this free software.  For both users' and
+authors' sake, the GPL requires that modified versions be marked as
+changed, so that their problems will not be attributed erroneously to
+authors of previous versions.
+
+  Some devices are designed to deny users access to install or run
+modified versions of the software inside them, although the manufacturer
+can do so.  This is fundamentally incompatible with the aim of
+protecting users' freedom to change the software.  The systematic
+pattern of such abuse occurs in the area of products for individuals to
+use, which is precisely where it is most unacceptable.  Therefore, we
+have designed this version of the GPL to prohibit the practice for those
+products.  If such problems arise substantially in other domains, we
+stand ready to extend this provision to those domains in future versions
+of the GPL, as needed to protect the freedom of users.
+
+  Finally, every program is threatened constantly by software patents.
+States should not allow patents to restrict development and use of
+software on general-purpose computers, but in those that do, we wish to
+avoid the special danger that patents applied to a free program could
+make it effectively proprietary.  To prevent this, the GPL assures that
+patents cannot be used to render the program non-free.
+
+  The precise terms and conditions for copying, distribution and
+modification follow.
+
+                       TERMS AND CONDITIONS
+
+  0. Definitions.
+
+  "This License" refers to version 3 of the GNU General Public License.
+
+  "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+  "The Program" refers to any copyrightable work licensed under this
+License.  Each licensee is addressed as "you".  "Licensees" and
+"recipients" may be individuals or organizations.
+
+  To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy.  The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+  A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+  To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy.  Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+  To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies.  Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+  An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License.  If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+  1. Source Code.
+
+  The "source code" for a work means the preferred form of the work
+for making modifications to it.  "Object code" means any non-source
+form of a work.
+
+  A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+  The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form.  A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+  The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities.  However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work.  For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+  The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+  The Corresponding Source for a work in source code form is that
+same work.
+
+  2. Basic Permissions.
+
+  All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met.  This License explicitly affirms your unlimited
+permission to run the unmodified Program.  The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work.  This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+  You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force.  You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright.  Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+  Conveying under any other circumstances is permitted solely under
+the conditions stated below.  Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+  3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+  No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+  When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+  4. Conveying Verbatim Copies.
+
+  You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+  You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+  5. Conveying Modified Source Versions.
+
+  You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+    a) The work must carry prominent notices stating that you modified
+    it, and giving a relevant date.
+
+    b) The work must carry prominent notices stating that it is
+    released under this License and any conditions added under section
+    7.  This requirement modifies the requirement in section 4 to
+    "keep intact all notices".
+
+    c) You must license the entire work, as a whole, under this
+    License to anyone who comes into possession of a copy.  This
+    License will therefore apply, along with any applicable section 7
+    additional terms, to the whole of the work, and all its parts,
+    regardless of how they are packaged.  This License gives no
+    permission to license the work in any other way, but it does not
+    invalidate such permission if you have separately received it.
+
+    d) If the work has interactive user interfaces, each must display
+    Appropriate Legal Notices; however, if the Program has interactive
+    interfaces that do not display Appropriate Legal Notices, your
+    work need not make them do so.
+
+  A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit.  Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+  6. Conveying Non-Source Forms.
+
+  You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+    a) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by the
+    Corresponding Source fixed on a durable physical medium
+    customarily used for software interchange.
+
+    b) Convey the object code in, or embodied in, a physical product
+    (including a physical distribution medium), accompanied by a
+    written offer, valid for at least three years and valid for as
+    long as you offer spare parts or customer support for that product
+    model, to give anyone who possesses the object code either (1) a
+    copy of the Corresponding Source for all the software in the
+    product that is covered by this License, on a durable physical
+    medium customarily used for software interchange, for a price no
+    more than your reasonable cost of physically performing this
+    conveying of source, or (2) access to copy the
+    Corresponding Source from a network server at no charge.
+
+    c) Convey individual copies of the object code with a copy of the
+    written offer to provide the Corresponding Source.  This
+    alternative is allowed only occasionally and noncommercially, and
+    only if you received the object code with such an offer, in accord
+    with subsection 6b.
+
+    d) Convey the object code by offering access from a designated
+    place (gratis or for a charge), and offer equivalent access to the
+    Corresponding Source in the same way through the same place at no
+    further charge.  You need not require recipients to copy the
+    Corresponding Source along with the object code.  If the place to
+    copy the object code is a network server, the Corresponding Source
+    may be on a different server (operated by you or a third party)
+    that supports equivalent copying facilities, provided you maintain
+    clear directions next to the object code saying where to find the
+    Corresponding Source.  Regardless of what server hosts the
+    Corresponding Source, you remain obligated to ensure that it is
+    available for as long as needed to satisfy these requirements.
+
+    e) Convey the object code using peer-to-peer transmission, provided
+    you inform other peers where the object code and Corresponding
+    Source of the work are being offered to the general public at no
+    charge under subsection 6d.
+
+  A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+  A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling.  In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage.  For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product.  A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+  "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source.  The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+  If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information.  But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+  The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed.  Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+  Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+  7. Additional Terms.
+
+  "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law.  If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+  When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it.  (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.)  You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+  Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+    a) Disclaiming warranty or limiting liability differently from the
+    terms of sections 15 and 16 of this License; or
+
+    b) Requiring preservation of specified reasonable legal notices or
+    author attributions in that material or in the Appropriate Legal
+    Notices displayed by works containing it; or
+
+    c) Prohibiting misrepresentation of the origin of that material, or
+    requiring that modified versions of such material be marked in
+    reasonable ways as different from the original version; or
+
+    d) Limiting the use for publicity purposes of names of licensors or
+    authors of the material; or
+
+    e) Declining to grant rights under trademark law for use of some
+    trade names, trademarks, or service marks; or
+
+    f) Requiring indemnification of licensors and authors of that
+    material by anyone who conveys the material (or modified versions of
+    it) with contractual assumptions of liability to the recipient, for
+    any liability that these contractual assumptions directly impose on
+    those licensors and authors.
+
+  All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10.  If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term.  If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+  If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+  Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+  8. Termination.
+
+  You may not propagate or modify a covered work except as expressly
+provided under this License.  Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+  However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+  Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+  Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License.  If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+  9. Acceptance Not Required for Having Copies.
+
+  You are not required to accept this License in order to receive or
+run a copy of the Program.  Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance.  However,
+nothing other than this License grants you permission to propagate or
+modify any covered work.  These actions infringe copyright if you do
+not accept this License.  Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+  10. Automatic Licensing of Downstream Recipients.
+
+  Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License.  You are not responsible
+for enforcing compliance by third parties with this License.
+
+  An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations.  If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+  You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License.  For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+  11. Patents.
+
+  A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based.  The
+work thus licensed is called the contributor's "contributor version".
+
+  A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version.  For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+  Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+  In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement).  To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+  If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients.  "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+  If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+  A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License.  You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+  Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+  12. No Surrender of Others' Freedom.
+
+  If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License.  If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all.  For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+  13. Use with the GNU Affero General Public License.
+
+  Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU Affero General Public License into a single
+combined work, and to convey the resulting work.  The terms of this
+License will continue to apply to the part which is the covered work,
+but the special requirements of the GNU Affero General Public License,
+section 13, concerning interaction through a network will apply to the
+combination as such.
+
+  14. Revised Versions of this License.
+
+  The Free Software Foundation may publish revised and/or new versions of
+the GNU General Public License from time to time.  Such new versions will
+be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+  Each version is given a distinguishing version number.  If the
+Program specifies that a certain numbered version of the GNU General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation.  If the Program does not specify a version number of the
+GNU General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+  If the Program specifies that a proxy can decide which future
+versions of the GNU General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+  Later license versions may give you additional or different
+permissions.  However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+  15. Disclaimer of Warranty.
+
+  THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+  16. Limitation of Liability.
+
+  IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+  17. Interpretation of Sections 15 and 16.
+
+  If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+                     END OF TERMS AND CONDITIONS
+
+            How to Apply These Terms to Your New Programs
+
+  If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+  To do so, attach the following notices to the program.  It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+    <one line to give the program's name and a brief idea of what it does.>
+    Copyright (C) <year>  <name of author>
+
+    This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+Also add information on how to contact you by electronic and paper mail.
+
+  If the program does terminal interaction, make it output a short
+notice like this when it starts in an interactive mode:
+
+    <program>  Copyright (C) <year>  <name of author>
+    This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
+    This is free software, and you are welcome to redistribute it
+    under certain conditions; type `show c' for details.
+
+The hypothetical commands `show w' and `show c' should show the appropriate
+parts of the General Public License.  Of course, your program's commands
+might be different; for a GUI interface, you would use an "about box".
+
+  You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU GPL, see
+<https://www.gnu.org/licenses/>.
+
+  The GNU General Public License does not permit incorporating your program
+into proprietary programs.  If your program is a subroutine library, you
+may consider it more useful to permit linking proprietary applications with
+the library.  If this is what you want to do, use the GNU Lesser General
+Public License instead of this License.  But first, please read
+<https://www.gnu.org/licenses/why-not-lgpl.html>.

README.md

@@ -0,0 +1,105 @@
+# WlxOverlay-S
+
+A lightweight OpenXR/OpenVR overlay for Wayland and X11 desktops, inspired by XSOverlay.
+
+WlxOverlay-S allows you to access your desktop screens while in VR.
+
+Compared to similar software, WlxOverlay-S aims to run alongside your other VR games or experiences and have as little performance impact as possible. The UI looks and rendering methods are kept simple and efficient as much as possible.
+
+This is the coming-together of two of my previous projects:
+- [WlxOverlay](https://github.com/galister/WlxOverlay) (SteamVR overlay written in C#)
+- [WlxOverlay-X](https://github.com/galister/wlx-overlay-x) (OpenXR overlay using StereoKit, written in Rust)
+
+# Join the Linux VR Community
+
+We are available on either:
+- Discord: https://discord.gg/gHwJ2vwSWV
+- Matrix Space: `#linux-vr-adventures:matrix.org`
+
+Questions/issues specific to WlxOverlay-S will be handled in the `wlxoverlay` chat room.
+
+# Setup
+
+1. Grab the latest AppImage from [Releases](https://github.com/galister/wlx-overlay-s/releases).
+1. `chmod +x WlxOverlay-S-*.AppImage`
+1. Start Monado, WiVRn or SteamVR.
+1. Run the overlay
+
+AUR package is [wlx-overlay-s-git](https://aur.archlinux.org/packages/wlx-overlay-s-git).
+
+You may also want to [build from source](https://github.com/galister/wlx-overlay-s/wiki/Building-from-Source).
+
+# First Start
+
+**If you get a screen share pop-up, check the terminal and select the screens in the order it tells you to.**
+
+SteamVR users: WlxOverlay-S will register itself for auto-start, so you will not need to start it every time.
+
+**Please continue reading the guide below.**
+
+# Getting Started
+
+### Pointer Modes AKA Laser Colors
+
+Much of the functionality in WlxOverlay-S depends on what color of laser you are using to interact with a UI element. \
+Using the default settings, there are 3 modes:
+- Regular Mode: Blue laser
+- Right-click Mode: Orange laser
+- Middle-click Mode: Purple laser
+
+Please see the bindings section below on how to activate these modes.
+
+The guide here uses the colors for ease of getting started.
+
+### The Watch
+
+Check your left wrist for the watch. The watch is your primary tool for controlling the app.
+
+![Watch usage guide](https://github.com/galister/wlx-overlay-s/blob/guide/wlx-watch.png)
+
+### The Screens
+
+Hovering a pointer over a screen will move the mouse. If there are more than one pointers hovering a screen, the pointer that was last used to click will take precedence.
+
+The click depends on the laser color:
+- Blue laser: Left click
+- Orange laser: Right click
+- Purple laser: Middle click
+- Stick up/down: Scroll wheel
+
+See the bindings section on how to grab, move and resize screens.
+
+### The keyboard
+
+The keyboard is fully customizable via the [keyboard.yaml](https://raw.githubusercontent.com/galister/wlx-overlay-s/main/src/res/keyboard.yaml) file. \
+Download it into your `~/.config/wlxoverlay/` folder and edit it to your liking.
+
+Typing
+- Use the BLUE laser when typing regularly.
+- While using ORANGE laser, all keystrokes will have SHIFT applied.
+- Purple laser has no effect as of now.
+
+**Modifier Keys** are sticky. They will remain pressed until you press a non-modifier key, or toggle them off.
+
+### Default Bindings
+
+![Index Controller Bindings](https://github.com/galister/wlx-overlay-s/blob/guide/wlx-index.png)
+
+![Touch Controller Bindings](https://github.com/galister/wlx-overlay-s/blob/guide/wlx-oculus.png)
+
+If your bindings are not supported, please reach out. \
+We would like to work with you and include additional bindings.
+
+# Known Issues
+
+## Modifiers get stuck in weird ways
+
+This is a rare issue that can make your desktop not react to click or keys due to a modifier being stuck somewhere. Restarting the overlay fixes this.
+
+## Scroll wheel doesn't work
+
+This seems to be an issue specific to Electron apps (Discord, Element, Slack, Spotify) on Wayland. Scrolling will work when using these in your web browser.
+
+## X11 limitations
+
+DPI scaling and upright screens are not supported on X11. These might display incorrectly or mess up your mouse position.

com.github.galister.wlx-overlay-s.json

@@ -1,15 +0,0 @@
-{
-  "appid": "com.github.galister.wlx-overlay-s",
-  "name": "WayVR",
-  "version": "26.2.1",
-  "hompage_url": "https://github.com/wlx-team/wayvr",
-  "icon_url": "https://raw.githubusercontent.com/wlx-team/wayvr/refs/heads/main/wayvr/wayvr.svg",
-  "screenshots": [
-    "https://github.com/wlx-team/wayvr/raw/guide/wayvr-readme-screenshot.webp?raw=true"
-  ],
-  "description": "A lightweight OpenXR/OpenVR overlay for Wayland and X11 desktops, inspired by XSOverlay.\n\nWayVR allows you to access your desktop screens while in VR.\n\nIn comparison to similar overlays, WayVR aims to run alongside VR games and experiences while having as little performance impact as possible. The UI appearance and rendering techniques are kept as simple and efficient as possible, while still allowing a high degree of customizability.",
-  "short_description": "Access your Wayland/X11 desktop",
-  "exec_url": "https://github.com/wlx-team/wayvr/releases/download/v26.2.1/WayVR-v26.2.1-x86_64.AppImage",
-  "luanch_opts": ["--openxr"],
-  "args": ["--openxr"]
-}

src/backend/common.rs

@@ -0,0 +1,331 @@
+use std::{
+    collections::{BinaryHeap, VecDeque},
+    f32::consts::PI,
+    sync::Arc,
+    time::Instant,
+};
+
+#[cfg(feature = "openxr")]
+use openxr as xr;
+
+use glam::{Affine3A, Vec2, Vec3A, Vec3Swizzles};
+use idmap::IdMap;
+use serde::Deserialize;
+use thiserror::Error;
+
+use crate::{
+    overlays::{
+        keyboard::create_keyboard,
+        watch::{create_watch, WATCH_NAME},
+    },
+    state::{AppState, ScreenMeta},
+};
+
+use super::overlay::{OverlayBackend, OverlayData, OverlayState};
+
+#[derive(Error, Debug)]
+pub enum BackendError {
+    #[error("backend not supported")]
+    NotSupported,
+    #[cfg(feature = "openxr")]
+    #[error("OpenXR Error: {0:?}")]
+    OpenXrError(#[from] xr::sys::Result),
+    #[error("Shutdown")]
+    Shutdown,
+    #[error("Restart")]
+    Restart,
+    #[error("Fatal: {0:?}")]
+    Fatal(#[from] anyhow::Error),
+}
+
+pub struct OverlayContainer<T>
+where
+    T: Default,
+{
+    overlays: IdMap<usize, OverlayData<T>>,
+    pub extent: Vec2,
+}
+
+impl<T> OverlayContainer<T>
+where
+    T: Default,
+{
+    pub fn new(app: &mut AppState) -> anyhow::Result<Self> {
+        let mut overlays = IdMap::new();
+        let (screens, extent) = if std::env::var("WAYLAND_DISPLAY").is_ok() {
+            crate::overlays::screen::get_screens_wayland(&app.session)?
+        } else {
+            crate::overlays::screen::get_screens_x11(&app.session)?
+        };
+
+        app.screens.clear();
+        for screen in screens.iter() {
+            app.screens.push(ScreenMeta {
+                name: screen.state.name.clone(),
+                id: screen.state.id,
+            });
+        }
+
+        let mut watch = create_watch::<T>(app)?;
+        watch.state.want_visible = true;
+        overlays.insert(watch.state.id, watch);
+
+        let mut keyboard = create_keyboard(app)?;
+        keyboard.state.show_hide = true;
+        keyboard.state.want_visible = false;
+        overlays.insert(keyboard.state.id, keyboard);
+
+        let mut show_screens = app.session.config.show_screens.clone();
+        if show_screens.is_empty() {
+            if let Some(s) = screens.first() {
+                show_screens.push(s.state.name.clone());
+            }
+        }
+
+        for mut screen in screens {
+            if show_screens.contains(&screen.state.name) {
+                screen.state.show_hide = true;
+                screen.state.want_visible = false;
+            }
+            overlays.insert(screen.state.id, screen);
+        }
+        Ok(Self { overlays, extent })
+    }
+
+    pub fn mut_by_selector(&mut self, selector: &OverlaySelector) -> Option<&mut OverlayData<T>> {
+        match selector {
+            OverlaySelector::Id(id) => self.mut_by_id(*id),
+            OverlaySelector::Name(name) => self.mut_by_name(name),
+        }
+    }
+
+    pub fn remove_by_selector(&mut self, selector: &OverlaySelector) -> Option<OverlayData<T>> {
+        match selector {
+            OverlaySelector::Id(id) => self.overlays.remove(id),
+            OverlaySelector::Name(name) => {
+                let id = self
+                    .overlays
+                    .iter()
+                    .find(|(_, o)| *o.state.name == **name)
+                    .map(|(id, _)| *id);
+                id.and_then(|id| self.overlays.remove(id))
+            }
+        }
+    }
+
+    pub fn get_by_id(&mut self, id: usize) -> Option<&OverlayData<T>> {
+        self.overlays.get(id)
+    }
+
+    pub fn mut_by_id(&mut self, id: usize) -> Option<&mut OverlayData<T>> {
+        self.overlays.get_mut(id)
+    }
+
+    pub fn get_by_name<'a>(&'a mut self, name: &str) -> Option<&'a OverlayData<T>> {
+        self.overlays.values().find(|o| *o.state.name == *name)
+    }
+
+    pub fn mut_by_name<'a>(&'a mut self, name: &str) -> Option<&'a mut OverlayData<T>> {
+        self.overlays.values_mut().find(|o| *o.state.name == *name)
+    }
+
+    pub fn iter(&self) -> impl Iterator<Item = &'_ OverlayData<T>> {
+        self.overlays.values()
+    }
+
+    pub fn iter_mut(&mut self) -> impl Iterator<Item = &'_ mut OverlayData<T>> {
+        self.overlays.values_mut()
+    }
+
+    pub fn add(&mut self, overlay: OverlayData<T>) {
+        self.overlays.insert(overlay.state.id, overlay);
+    }
+
+    pub fn show_hide(&mut self, app: &mut AppState) {
+        let any_shown = self
+            .overlays
+            .values()
+            .any(|o| o.state.show_hide && o.state.want_visible);
+
+        self.overlays.values_mut().for_each(|o| {
+            if o.state.show_hide {
+                o.state.want_visible = !any_shown;
+                if o.state.want_visible && o.state.recenter {
+                    o.state.reset(app, false);
+                }
+            }
+            // toggle watch back on if it was hidden
+            if !any_shown && *o.state.name == *WATCH_NAME {
+                o.state.reset(app, true);
+            }
+        })
+    }
+}
+
+#[derive(Clone, Deserialize)]
+#[serde(untagged)]
+pub enum OverlaySelector {
+    Id(usize),
+    Name(Arc<str>),
+}
+
+struct AppTask {
+    pub not_before: Instant,
+    pub task: TaskType,
+}
+
+impl PartialEq<AppTask> for AppTask {
+    fn eq(&self, other: &Self) -> bool {
+        self.not_before == other.not_before
+    }
+}
+impl PartialOrd<AppTask> for AppTask {
+    fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
+        Some(self.cmp(other))
+    }
+}
+impl Eq for AppTask {}
+impl Ord for AppTask {
+    fn cmp(&self, other: &Self) -> std::cmp::Ordering {
+        self.not_before.cmp(&other.not_before).reverse()
+    }
+}
+
+pub enum SystemTask {
+    ColorGain(ColorChannel, f32),
+    ResetPlayspace,
+    FixFloor,
+}
+
+pub type OverlayTask = dyn FnOnce(&mut AppState, &mut OverlayState) + Send;
+pub type CreateOverlayTask =
+    dyn FnOnce(&mut AppState) -> Option<(OverlayState, Box<dyn OverlayBackend>)> + Send;
+
+pub enum TaskType {
+    Global(Box<dyn FnOnce(&mut AppState) + Send>),
+    Overlay(OverlaySelector, Box<OverlayTask>),
+    CreateOverlay(OverlaySelector, Box<CreateOverlayTask>),
+    DropOverlay(OverlaySelector),
+    System(SystemTask),
+}
+
+#[derive(Deserialize, Clone, Copy)]
+pub enum ColorChannel {
+    R,
+    G,
+    B,
+    All,
+}
+
+pub struct TaskContainer {
+    tasks: BinaryHeap<AppTask>,
+}
+
+impl TaskContainer {
+    pub fn new() -> Self {
+        Self {
+            tasks: BinaryHeap::new(),
+        }
+    }
+
+    pub fn enqueue(&mut self, task: TaskType) {
+        self.tasks.push(AppTask {
+            not_before: Instant::now(),
+            task,
+        });
+    }
+
+    pub fn enqueue_at(&mut self, task: TaskType, not_before: Instant) {
+        self.tasks.push(AppTask { not_before, task });
+    }
+
+    pub fn retrieve_due(&mut self, dest_buf: &mut VecDeque<TaskType>) {
+        let now = Instant::now();
+
+        while let Some(task) = self.tasks.peek() {
+            if task.not_before > now {
+                break;
+            }
+
+            // Safe unwrap because we peeked.
+            dest_buf.push_back(self.tasks.pop().unwrap().task);
+        }
+    }
+}
+
+pub fn raycast_plane(
+    source: &Affine3A,
+    source_fwd: Vec3A,
+    plane: &Affine3A,
+    plane_norm: Vec3A,
+) -> Option<(f32, Vec2)> {
+    let plane_normal = plane.transform_vector3a(plane_norm);
+    let ray_dir = source.transform_vector3a(source_fwd);
+
+    let d = plane.translation.dot(-plane_normal);
+    let dist = -(d + source.translation.dot(plane_normal)) / ray_dir.dot(plane_normal);
+
+    let hit_local = plane
+        .inverse()
+        .transform_point3a(source.translation + ray_dir * dist)
+        .xy();
+
+    Some((dist, hit_local))
+}
+
+pub fn raycast_cylinder(
+    source: &Affine3A,
+    source_fwd: Vec3A,
+    plane: &Affine3A,
+    curvature: f32,
+) -> Option<(f32, Vec2)> {
+    // this is solved locally; (0,0) is the center of the cylinder, and the cylinder is aligned with the Y axis
+    let size = plane.x_axis.length();
+    let to_local = Affine3A {
+        matrix3: plane.matrix3.mul_scalar(1.0 / size),
+        translation: plane.translation,
+    }
+    .inverse();
+
+    let r = size / (2.0 * PI * curvature);
+
+    let ray_dir = to_local.transform_vector3a(source.transform_vector3a(source_fwd));
+    let ray_origin = to_local.transform_point3a(source.translation) + Vec3A::NEG_Z * r;
+
+    let d = ray_dir.xz();
+    let s = ray_origin.xz();
+
+    let a = d.dot(d);
+    let b = d.dot(s);
+    let c = s.dot(s) - r * r;
+
+    let d = (b * b) - (a * c);
+    if d < f32::EPSILON {
+        return None;
+    }
+
+    let sqrt_d = d.sqrt();
+
+    let t1 = (-b - sqrt_d) / a;
+    let t2 = (-b + sqrt_d) / a;
+
+    let t = t1.max(t2);
+
+    if t < f32::EPSILON {
+        return None;
+    }
+
+    let mut hit_local = ray_origin + ray_dir * t;
+    if hit_local.z > 0.0 {
+        // hitting the opposite half of the cylinder
+        return None;
+    }
+
+    let max_angle = 2.0 * (size / (2.0 * r));
+    let x_angle = (hit_local.x / r).asin();
+
+    hit_local.x = x_angle / max_angle;
+    hit_local.y /= size;
+
+    Some((t, hit_local.xy()))
+}

src/backend/input.rs

@@ -0,0 +1,537 @@
+use std::{collections::VecDeque, time::Instant};
+
+use glam::{Affine3A, Vec2, Vec3A};
+
+#[cfg(feature = "openvr")]
+use ovr_overlay::TrackedDeviceIndex;
+use smallvec::{smallvec, SmallVec};
+
+use crate::state::AppState;
+
+use super::{
+    common::{raycast_cylinder, raycast_plane, OverlayContainer},
+    overlay::OverlayData,
+};
+
+pub struct TrackedDevice {
+    #[cfg(feature = "openvr")]
+    pub index: TrackedDeviceIndex,
+    pub valid: bool,
+    pub soc: Option<f32>,
+    pub charging: bool,
+    pub role: TrackedDeviceRole,
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum TrackedDeviceRole {
+    None,
+    Hmd,
+    LeftHand,
+    RightHand,
+    Tracker,
+}
+
+pub struct InputState {
+    pub hmd: Affine3A,
+    pub pointers: [Pointer; 2],
+    pub devices: Vec<TrackedDevice>,
+}
+
+impl InputState {
+    pub fn new() -> Self {
+        Self {
+            hmd: Affine3A::IDENTITY,
+            pointers: [Pointer::new(0), Pointer::new(1)],
+            devices: Vec::new(),
+        }
+    }
+
+    pub fn pre_update(&mut self) {
+        self.pointers[0].before = self.pointers[0].now;
+        self.pointers[1].before = self.pointers[1].now;
+    }
+
+    pub fn post_update(&mut self) {
+        for hand in &mut self.pointers {
+            #[cfg(debug_assertions)]
+            {
+                if hand.now.click != hand.before.click {
+                    log::debug!("Hand {}: click {}", hand.idx, hand.now.click);
+                }
+                if hand.now.grab != hand.before.grab {
+                    log::debug!("Hand {}: grab {}", hand.idx, hand.now.grab);
+                }
+                if hand.now.alt_click != hand.before.alt_click {
+                    log::debug!("Hand {}: alt_click {}", hand.idx, hand.now.alt_click);
+                }
+                if hand.now.show_hide != hand.before.show_hide {
+                    log::debug!("Hand {}: show_hide {}", hand.idx, hand.now.show_hide);
+                }
+                if hand.now.space_drag != hand.before.space_drag {
+                    log::debug!("Hand {}: space_drag {}", hand.idx, hand.now.space_drag);
+                }
+                if hand.now.space_rotate != hand.before.space_rotate {
+                    log::debug!("Hand {}: space_rotate {}", hand.idx, hand.now.space_rotate);
+                }
+                if hand.now.click_modifier_right != hand.before.click_modifier_right {
+                    log::debug!(
+                        "Hand {}: click_modifier_right {}",
+                        hand.idx,
+                        hand.now.click_modifier_right
+                    );
+                }
+                if hand.now.click_modifier_middle != hand.before.click_modifier_middle {
+                    log::debug!(
+                        "Hand {}: click_modifier_middle {}",
+                        hand.idx,
+                        hand.now.click_modifier_middle
+                    );
+                }
+            }
+
+            if hand.now.click {
+                hand.last_click = Instant::now();
+            }
+
+            if hand.now.click_modifier_right {
+                hand.interaction.mode = PointerMode::Right;
+                continue;
+            }
+
+            if hand.now.click_modifier_middle {
+                hand.interaction.mode = PointerMode::Middle;
+                continue;
+            }
+
+            let hmd_up = self.hmd.transform_vector3a(Vec3A::Y);
+            let dot =
+                hmd_up.dot(hand.pose.transform_vector3a(Vec3A::X)) * (1.0 - 2.0 * hand.idx as f32);
+
+            hand.interaction.mode = if dot < -0.85 {
+                PointerMode::Right
+            } else if dot > 0.7 {
+                PointerMode::Middle
+            } else {
+                PointerMode::Left
+            };
+
+            let middle_click_orientation = false;
+            let right_click_orientation = false;
+            match hand.interaction.mode {
+                PointerMode::Middle => {
+                    if !middle_click_orientation {
+                        hand.interaction.mode = PointerMode::Left;
+                    }
+                }
+                PointerMode::Right => {
+                    if !right_click_orientation {
+                        hand.interaction.mode = PointerMode::Left;
+                    }
+                }
+                _ => {}
+            };
+        }
+    }
+}
+
+pub struct InteractionState {
+    pub mode: PointerMode,
+    pub grabbed: Option<GrabData>,
+    pub clicked_id: Option<usize>,
+    pub hovered_id: Option<usize>,
+    pub release_actions: VecDeque<Box<dyn Fn()>>,
+    pub next_push: Instant,
+    pub haptics: Option<f32>,
+}
+
+impl Default for InteractionState {
+    fn default() -> Self {
+        Self {
+            mode: PointerMode::Left,
+            grabbed: None,
+            clicked_id: None,
+            hovered_id: None,
+            release_actions: VecDeque::new(),
+            next_push: Instant::now(),
+            haptics: None,
+        }
+    }
+}
+
+pub struct Pointer {
+    pub idx: usize,
+    pub pose: Affine3A,
+    pub raw_pose: Affine3A,
+    pub now: PointerState,
+    pub before: PointerState,
+    pub last_click: Instant,
+    pub(super) interaction: InteractionState,
+}
+
+impl Pointer {
+    pub fn new(idx: usize) -> Self {
+        debug_assert!(idx == 0 || idx == 1);
+        Self {
+            idx,
+            pose: Affine3A::IDENTITY,
+            raw_pose: Affine3A::IDENTITY,
+            now: Default::default(),
+            before: Default::default(),
+            last_click: Instant::now(),
+            interaction: Default::default(),
+        }
+    }
+}
+
+#[derive(Clone, Copy, Default)]
+pub struct PointerState {
+    pub scroll: f32,
+    pub click: bool,
+    pub grab: bool,
+    pub alt_click: bool,
+    pub show_hide: bool,
+    pub space_drag: bool,
+    pub space_rotate: bool,
+    pub click_modifier_right: bool,
+    pub click_modifier_middle: bool,
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct PointerHit {
+    pub pointer: usize,
+    pub overlay: usize,
+    pub mode: PointerMode,
+    pub primary: bool,
+    pub uv: Vec2,
+    pub dist: f32,
+}
+
+pub struct Haptics {
+    pub intensity: f32,
+    pub duration: f32,
+    pub frequency: f32,
+}
+
+pub trait InteractionHandler {
+    fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit) -> Option<Haptics>;
+    fn on_left(&mut self, app: &mut AppState, pointer: usize);
+    fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool);
+    fn on_scroll(&mut self, app: &mut AppState, hit: &PointerHit, delta: f32);
+}
+
+pub struct DummyInteractionHandler;
+
+impl InteractionHandler for DummyInteractionHandler {
+    fn on_left(&mut self, _app: &mut AppState, _pointer: usize) {}
+    fn on_hover(&mut self, _app: &mut AppState, _hit: &PointerHit) -> Option<Haptics> {
+        None
+    }
+    fn on_pointer(&mut self, _app: &mut AppState, _hit: &PointerHit, _pressed: bool) {}
+    fn on_scroll(&mut self, _app: &mut AppState, _hit: &PointerHit, _delta: f32) {}
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+struct RayHit {
+    overlay: usize,
+    global_pos: Vec3A,
+    local_pos: Vec2,
+    dist: f32,
+}
+
+#[derive(Debug, Clone, Copy, Default)]
+pub struct GrabData {
+    pub offset: Vec3A,
+    pub grabbed_id: usize,
+}
+
+#[repr(u8)]
+#[derive(Debug, Clone, Copy, Default)]
+pub enum PointerMode {
+    #[default]
+    Left,
+    Right,
+    Middle,
+    Special,
+}
+
+pub fn interact<O>(
+    overlays: &mut OverlayContainer<O>,
+    app: &mut AppState,
+) -> [(f32, Option<Haptics>); 2]
+where
+    O: Default,
+{
+    if app.input_state.pointers[1].last_click > app.input_state.pointers[0].last_click {
+        let right = interact_hand(1, overlays, app);
+        let left = interact_hand(0, overlays, app);
+        [left, right]
+    } else {
+        let left = interact_hand(0, overlays, app);
+        let right = interact_hand(1, overlays, app);
+        [left, right]
+    }
+}
+
+fn interact_hand<O>(
+    idx: usize,
+    overlays: &mut OverlayContainer<O>,
+    app: &mut AppState,
+) -> (f32, Option<Haptics>)
+where
+    O: Default,
+{
+    let hmd = &app.input_state.hmd;
+    let mut pointer = &mut app.input_state.pointers[idx];
+    if let Some(grab_data) = pointer.interaction.grabbed {
+        if let Some(grabbed) = overlays.mut_by_id(grab_data.grabbed_id) {
+            pointer.handle_grabbed(grabbed, hmd);
+        } else {
+            log::warn!("Grabbed overlay {} does not exist", grab_data.grabbed_id);
+            pointer.interaction.grabbed = None;
+        }
+        return (0.1, None);
+    }
+
+    let Some(mut hit) = pointer.get_nearest_hit(overlays) else {
+        if let Some(hovered_id) = pointer.interaction.hovered_id.take() {
+            if let Some(hovered) = overlays.mut_by_id(hovered_id) {
+                hovered.backend.on_left(app, idx);
+            }
+            pointer = &mut app.input_state.pointers[idx];
+            pointer.interaction.hovered_id = None;
+        }
+        if let Some(clicked_id) = pointer.interaction.clicked_id.take() {
+            if let Some(clicked) = overlays.mut_by_id(clicked_id) {
+                let hit = PointerHit {
+                    pointer: pointer.idx,
+                    overlay: clicked_id,
+                    mode: pointer.interaction.mode,
+                    ..Default::default()
+                };
+                clicked.backend.on_pointer(app, &hit, false);
+            }
+        }
+        return (0.0, None); // no hit
+    };
+
+    if let Some(hovered_id) = pointer.interaction.hovered_id {
+        if hovered_id != hit.overlay {
+            if let Some(old_hovered) = overlays.mut_by_id(hovered_id) {
+                if Some(pointer.idx) == old_hovered.primary_pointer {
+                    old_hovered.primary_pointer = None;
+                }
+                old_hovered.backend.on_left(app, idx);
+                pointer = &mut app.input_state.pointers[idx];
+            }
+        }
+    }
+    let Some(hovered) = overlays.mut_by_id(hit.overlay) else {
+        log::warn!("Hit overlay {} does not exist", hit.overlay);
+        return (0.0, None); // no hit
+    };
+
+    pointer.interaction.hovered_id = Some(hit.overlay);
+
+    if let Some(primary_pointer) = hovered.primary_pointer {
+        if hit.pointer <= primary_pointer {
+            hovered.primary_pointer = Some(hit.pointer);
+            hit.primary = true;
+        }
+    } else {
+        hovered.primary_pointer = Some(hit.pointer);
+        hit.primary = true;
+    }
+
+    #[cfg(debug_assertions)]
+    log::trace!("Hit: {} {:?}", hovered.state.name, hit);
+
+    if pointer.now.grab && !pointer.before.grab && hovered.state.grabbable {
+        pointer.start_grab(hovered);
+        return (
+            hit.dist,
+            Some(Haptics {
+                intensity: 0.25,
+                duration: 0.1,
+                frequency: 0.1,
+            }),
+        );
+    }
+
+    let haptics = hovered.backend.on_hover(app, &hit);
+    pointer = &mut app.input_state.pointers[idx];
+
+    if pointer.now.scroll.abs() > 0.1 {
+        let scroll = pointer.now.scroll;
+        if app.input_state.pointers[1 - idx]
+            .interaction
+            .grabbed
+            .is_some_and(|x| x.grabbed_id == hit.overlay)
+        {
+            let is_portrait = hovered.view().is_some_and(|v| {
+                let extent = v.image().extent();
+                extent[0] >= extent[1]
+            });
+
+            if is_portrait {
+                let cur = hovered.state.curvature.unwrap_or(0.0);
+                let new = (cur - scroll * 0.01).min(0.35);
+                if new <= f32::EPSILON {
+                    hovered.state.curvature = None;
+                } else {
+                    hovered.state.curvature = Some(new);
+                }
+            }
+        } else {
+            hovered.backend.on_scroll(app, &hit, scroll);
+        }
+        pointer = &mut app.input_state.pointers[idx];
+    }
+
+    if pointer.now.click && !pointer.before.click {
+        pointer.interaction.clicked_id = Some(hit.overlay);
+        hovered.backend.on_pointer(app, &hit, true);
+    } else if !pointer.now.click && pointer.before.click {
+        if let Some(clicked_id) = pointer.interaction.clicked_id.take() {
+            if let Some(clicked) = overlays.mut_by_id(clicked_id) {
+                clicked.backend.on_pointer(app, &hit, false);
+            }
+        } else {
+            hovered.backend.on_pointer(app, &hit, false);
+        }
+    }
+    (hit.dist, haptics)
+}
+
+impl Pointer {
+    fn get_nearest_hit<O>(&mut self, overlays: &mut OverlayContainer<O>) -> Option<PointerHit>
+    where
+        O: Default,
+    {
+        let mut hits: SmallVec<[RayHit; 8]> = smallvec!();
+
+        for overlay in overlays.iter() {
+            if !overlay.state.want_visible || !overlay.state.interactable {
+                continue;
+            }
+
+            if let Some(hit) = self.ray_test(
+                overlay.state.id,
+                &overlay.state.transform,
+                &overlay.state.curvature,
+            ) {
+                if hit.dist.is_infinite() || hit.dist.is_nan() {
+                    continue;
+                }
+                hits.push(hit);
+            }
+        }
+
+        hits.sort_by(|a, b| a.dist.total_cmp(&b.dist));
+
+        for hit in hits.iter() {
+            let overlay = overlays.get_by_id(hit.overlay).unwrap(); // safe because we just got the id from the overlay
+
+            let uv = overlay
+                .state
+                .interaction_transform
+                .transform_point2(hit.local_pos);
+
+            if uv.x < 0.0 || uv.x > 1.0 || uv.y < 0.0 || uv.y > 1.0 {
+                continue;
+            }
+
+            return Some(PointerHit {
+                pointer: self.idx,
+                overlay: hit.overlay,
+                mode: self.interaction.mode,
+                primary: false,
+                uv,
+                dist: hit.dist,
+            });
+        }
+
+        None
+    }
+
+    fn start_grab<O>(&mut self, overlay: &mut OverlayData<O>)
+    where
+        O: Default,
+    {
+        let offset = self
+            .pose
+            .inverse()
+            .transform_point3a(overlay.state.transform.translation);
+
+        self.interaction.grabbed = Some(GrabData {
+            offset,
+            grabbed_id: overlay.state.id,
+        });
+        log::info!("Hand {}: grabbed {}", self.idx, overlay.state.name);
+    }
+
+    fn handle_grabbed<O>(&mut self, overlay: &mut OverlayData<O>, hmd: &Affine3A)
+    where
+        O: Default,
+    {
+        if self.now.grab {
+            if let Some(grab_data) = self.interaction.grabbed.as_mut() {
+                if self.now.click {
+                    self.interaction.mode = PointerMode::Special;
+                    let cur_scale = overlay.state.transform.x_axis.length();
+                    if cur_scale < 0.1 && self.now.scroll > 0.0 {
+                        return;
+                    }
+                    if cur_scale > 20. && self.now.scroll < 0.0 {
+                        return;
+                    }
+
+                    overlay.state.transform.matrix3 = overlay
+                        .state
+                        .transform
+                        .matrix3
+                        .mul_scalar(1.0 - 0.025 * self.now.scroll);
+                } else {
+                    grab_data.offset.z -= self.now.scroll * 0.05;
+                }
+                overlay.state.transform.translation = self.pose.transform_point3a(grab_data.offset);
+                overlay.state.realign(hmd);
+                overlay.state.dirty = true;
+            } else {
+                log::error!("Grabbed overlay {} does not exist", overlay.state.id);
+                self.interaction.grabbed = None;
+            }
+        } else {
+            overlay.state.saved_point = Some(
+                hmd.inverse()
+                    .transform_point3a(overlay.state.transform.translation),
+            );
+            self.interaction.grabbed = None;
+            log::info!("Hand {}: dropped {}", self.idx, overlay.state.name);
+        }
+    }
+
+    fn ray_test(
+        &self,
+        overlay: usize,
+        transform: &Affine3A,
+        curvature: &Option<f32>,
+    ) -> Option<RayHit> {
+        let (dist, local_pos) = match curvature {
+            Some(curvature) => raycast_cylinder(&self.pose, Vec3A::NEG_Z, transform, *curvature),
+            _ => raycast_plane(&self.pose, Vec3A::NEG_Z, transform, Vec3A::NEG_Z),
+        }?;
+
+        if dist < 0.0 {
+            // hit is behind us
+            return None;
+        }
+
+        Some(RayHit {
+            overlay,
+            global_pos: self.pose.transform_point3a(Vec3A::NEG_Z * dist),
+            local_pos,
+            dist,
+        })
+    }
+}

src/backend/mod.rs

@@ -0,0 +1,14 @@
+pub mod common;
+pub mod input;
+pub mod notifications;
+
+#[cfg(feature = "openvr")]
+pub mod openvr;
+
+#[cfg(feature = "openxr")]
+pub mod openxr;
+
+#[cfg(feature = "osc")]
+pub mod osc;
+
+pub mod overlay;

src/backend/notifications.rs

@@ -0,0 +1,255 @@
+use dbus::{blocking::Connection, channel::MatchingReceiver, message::MatchRule};
+use serde::{Deserialize, Serialize};
+use std::{
+    path::PathBuf,
+    sync::{
+        atomic::{AtomicBool, Ordering},
+        mpsc, Arc,
+    },
+    time::Duration,
+};
+
+use crate::{config::def_true, config_io, overlays::toast::Toast, state::AppState};
+
+pub struct NotificationManager {
+    rx_toast: mpsc::Receiver<Toast>,
+    tx_toast: mpsc::SyncSender<Toast>,
+    dbus_data: Option<Connection>,
+    running: Arc<AtomicBool>,
+}
+
+impl NotificationManager {
+    pub fn new() -> Self {
+        let (tx_toast, rx_toast) = mpsc::sync_channel(10);
+        Self {
+            rx_toast,
+            tx_toast,
+            dbus_data: None,
+            running: Arc::new(AtomicBool::new(true)),
+        }
+    }
+
+    pub fn submit_pending(&self, app: &mut AppState) {
+        if let Some(c) = &self.dbus_data {
+            let _ = c.process(Duration::ZERO);
+        }
+
+        if app.session.config.notifications_enabled {
+            self.rx_toast.try_iter().for_each(|toast| {
+                toast.submit(app);
+            });
+        } else {
+            // consume without submitting
+            self.rx_toast.try_iter().last();
+        }
+    }
+
+    pub fn run_dbus(&mut self) {
+        let c = match Connection::new_session() {
+            Ok(c) => c,
+            Err(e) => {
+                log::error!(
+                    "Failed to connect to dbus. Desktop notifications will not work. Cause: {:?}",
+                    e
+                );
+                return;
+            }
+        };
+
+        let mut rule = MatchRule::new_method_call();
+        rule.member = Some("Notify".into());
+        rule.interface = Some("org.freedesktop.Notifications".into());
+        rule.path = Some("/org/freedesktop/Notifications".into());
+        rule.eavesdrop = true;
+
+        let proxy = c.with_proxy(
+            "org.freedesktop.DBus",
+            "/org/freedesktop/DBus",
+            Duration::from_millis(5000),
+        );
+        let result: Result<(), dbus::Error> = proxy.method_call(
+            "org.freedesktop.DBus.Monitoring",
+            "BecomeMonitor",
+            (vec![rule.match_str()], 0u32),
+        );
+
+        match result {
+            Ok(_) => {
+                let sender = self.tx_toast.clone();
+                c.start_receive(
+                    rule,
+                    Box::new(move |msg, _| {
+                        if let Ok(toast) = parse_dbus(&msg) {
+                            match sender.try_send(toast) {
+                                Ok(_) => {}
+                                Err(e) => {
+                                    log::error!("Failed to send notification: {:?}", e);
+                                }
+                            }
+                        }
+                        true
+                    }),
+                );
+                log::info!("Listening to DBus notifications via BecomeMonitor.");
+            }
+            Err(_) => {
+                let rule_with_eavesdrop = {
+                    let mut rule = rule.clone();
+                    rule.eavesdrop = true;
+                    rule
+                };
+
+                let sender2 = self.tx_toast.clone();
+                let result = c.add_match(rule_with_eavesdrop, move |_: (), _, msg| {
+                    if let Ok(toast) = parse_dbus(msg) {
+                        match sender2.try_send(toast) {
+                            Ok(_) => {}
+                            Err(e) => {
+                                log::error!("Failed to send notification: {:?}", e);
+                            }
+                        }
+                    }
+                    true
+                });
+
+                match result {
+                    Ok(_) => {
+                        log::info!("Listening to DBus notifications via eavesdrop.");
+                    }
+                    Err(_) => {
+                        log::error!(
+                            "Failed to add DBus match. Desktop notifications will not work.",
+                        );
+                    }
+                }
+            }
+        }
+
+        self.dbus_data = Some(c);
+    }
+
+    pub fn run_udp(&mut self) {
+        let sender = self.tx_toast.clone();
+        let running = self.running.clone();
+        let _ = std::thread::spawn(move || {
+            let addr = "127.0.0.1:42069";
+            let socket = match std::net::UdpSocket::bind(addr) {
+                Ok(s) => s,
+                Err(e) => {
+                    log::error!("Failed to bind notification socket @ {}: {:?}", addr, e);
+                    return;
+                }
+            };
+            if let Err(err) = socket.set_read_timeout(Some(Duration::from_millis(200))) {
+                log::error!("Failed to set read timeout: {:?}", err);
+            }
+
+            let mut buf = [0u8; 1024 * 16]; // vrcx embeds icons as b64
+
+            while running.load(Ordering::Relaxed) {
+                if let Ok((num_bytes, _)) = socket.recv_from(&mut buf) {
+                    let json_str = match std::str::from_utf8(&buf[..num_bytes]) {
+                        Ok(s) => s,
+                        Err(e) => {
+                            log::error!("Failed to receive notification message: {:?}", e);
+                            continue;
+                        }
+                    };
+                    let msg = match serde_json::from_str::<XsoMessage>(json_str) {
+                        Ok(m) => m,
+                        Err(e) => {
+                            log::error!("Failed to parse notification message: {:?}", e);
+                            continue;
+                        }
+                    };
+
+                    if msg.messageType != 1 {
+                        continue;
+                    }
+
+                    let toast = Toast::new(msg.title, msg.content.unwrap_or_else(|| "".into()))
+                        .with_timeout(msg.timeout.unwrap_or(5.))
+                        .with_sound(msg.volume.unwrap_or(-1.) >= 0.); // XSOverlay still plays at 0,
+
+                    match sender.try_send(toast) {
+                        Ok(_) => {}
+                        Err(e) => {
+                            log::error!("Failed to send notification: {:?}", e);
+                        }
+                    }
+                }
+            }
+            log::info!("Notification listener stopped.");
+        });
+    }
+}
+
+impl Drop for NotificationManager {
+    fn drop(&mut self) {
+        self.running.store(false, Ordering::Relaxed);
+    }
+}
+
+fn parse_dbus(msg: &dbus::Message) -> anyhow::Result<Toast> {
+    let mut args = msg.iter_init();
+    let app_name: String = args.read()?;
+    let _replaces_id: u32 = args.read()?;
+    let _app_icon: String = args.read()?;
+    let summary: String = args.read()?;
+    let body: String = args.read()?;
+
+    let title = if summary.is_empty() {
+        app_name
+    } else {
+        summary
+    };
+
+    Ok(Toast::new(title.into(), body.into())
+        .with_timeout(5.0)
+        .with_opacity(1.0))
+    // leave the audio part to the desktop env
+}
+
+#[allow(non_snake_case)]
+#[derive(Debug, Deserialize)]
+struct XsoMessage {
+    messageType: i32,
+    index: Option<i32>,
+    volume: Option<f32>,
+    audioPath: Option<Arc<str>>,
+    timeout: Option<f32>,
+    title: Arc<str>,
+    content: Option<Arc<str>>,
+    icon: Option<Arc<str>>,
+    height: Option<f32>,
+    opacity: Option<f32>,
+    useBase64Icon: Option<bool>,
+    sourceApp: Option<Arc<str>>,
+    alwaysShow: Option<bool>,
+}
+
+#[derive(Deserialize, Serialize)]
+pub struct NotifiConf {
+    #[serde(default = "def_true")]
+    pub notifications_enabled: bool,
+
+    #[serde(default = "def_true")]
+    pub notifications_sound_enabled: bool,
+}
+
+fn get_config_path() -> PathBuf {
+    let mut path = config_io::get_conf_d_path();
+    path.push("notifications.yaml");
+    path
+}
+pub fn save_notifications(app: &mut AppState) -> anyhow::Result<()> {
+    let conf = NotifiConf {
+        notifications_enabled: app.session.config.notifications_enabled,
+        notifications_sound_enabled: app.session.config.notifications_sound_enabled,
+    };
+
+    let yaml = serde_yaml::to_string(&conf)?;
+    std::fs::write(get_config_path(), yaml)?;
+
+    Ok(())
+}

src/backend/openvr/helpers.rs

@@ -0,0 +1,189 @@
+use std::ffi::CStr;
+
+use glam::Affine3A;
+use ovr_overlay::{pose::Matrix3x4, settings::SettingsManager, sys::HmdMatrix34_t};
+use thiserror::Error;
+
+use crate::backend::common::{BackendError, ColorChannel};
+
+pub trait Affine3AConvert {
+    fn from_affine(affine: &Affine3A) -> Self;
+    fn to_affine(&self) -> Affine3A;
+}
+
+impl Affine3AConvert for Matrix3x4 {
+    fn from_affine(affine: &Affine3A) -> Self {
+        Matrix3x4([
+            [
+                affine.matrix3.x_axis.x,
+                affine.matrix3.y_axis.x,
+                affine.matrix3.z_axis.x,
+                affine.translation.x,
+            ],
+            [
+                affine.matrix3.x_axis.y,
+                affine.matrix3.y_axis.y,
+                affine.matrix3.z_axis.y,
+                affine.translation.y,
+            ],
+            [
+                affine.matrix3.x_axis.z,
+                affine.matrix3.y_axis.z,
+                affine.matrix3.z_axis.z,
+                affine.translation.z,
+            ],
+        ])
+    }
+
+    fn to_affine(&self) -> Affine3A {
+        Affine3A::from_cols_array_2d(&[
+            [self.0[0][0], self.0[1][0], self.0[2][0]],
+            [self.0[0][1], self.0[1][1], self.0[2][1]],
+            [self.0[0][2], self.0[1][2], self.0[2][2]],
+            [self.0[0][3], self.0[1][3], self.0[2][3]],
+        ])
+    }
+}
+
+impl Affine3AConvert for HmdMatrix34_t {
+    fn from_affine(affine: &Affine3A) -> Self {
+        HmdMatrix34_t {
+            m: [
+                [
+                    affine.matrix3.x_axis.x,
+                    affine.matrix3.y_axis.x,
+                    affine.matrix3.z_axis.x,
+                    affine.translation.x,
+                ],
+                [
+                    affine.matrix3.x_axis.y,
+                    affine.matrix3.y_axis.y,
+                    affine.matrix3.z_axis.y,
+                    affine.translation.y,
+                ],
+                [
+                    affine.matrix3.x_axis.z,
+                    affine.matrix3.y_axis.z,
+                    affine.matrix3.z_axis.z,
+                    affine.translation.z,
+                ],
+            ],
+        }
+    }
+
+    fn to_affine(&self) -> Affine3A {
+        Affine3A::from_cols_array_2d(&[
+            [self.m[0][0], self.m[1][0], self.m[2][0]],
+            [self.m[0][1], self.m[1][1], self.m[2][1]],
+            [self.m[0][2], self.m[1][2], self.m[2][2]],
+            [self.m[0][3], self.m[1][3], self.m[2][3]],
+        ])
+    }
+}
+
+#[derive(Error, Debug)]
+pub(super) enum OVRError {
+    #[error("ovr input error: {0}")]
+    InputError(&'static str),
+}
+
+impl From<ovr_overlay::errors::EVRInputError> for OVRError {
+    fn from(e: ovr_overlay::errors::EVRInputError) -> Self {
+        OVRError::InputError(e.description())
+    }
+}
+
+impl From<OVRError> for BackendError {
+    fn from(e: OVRError) -> Self {
+        BackendError::Fatal(anyhow::Error::new(e))
+    }
+}
+
+use cstr::cstr;
+const STEAMVR_SECTION: &CStr = cstr!("steamvr");
+const COLOR_GAIN_CSTR: [&CStr; 3] = [
+    cstr!("hmdDisplayColorGainR"),
+    cstr!("hmdDisplayColorGainG"),
+    cstr!("hmdDisplayColorGainB"),
+];
+
+pub(super) fn adjust_gain(
+    settings: &mut SettingsManager,
+    ch: ColorChannel,
+    delta: f32,
+) -> Option<()> {
+    let current = [
+        settings
+            .get_float(STEAMVR_SECTION, COLOR_GAIN_CSTR[0])
+            .ok()?,
+        settings
+            .get_float(STEAMVR_SECTION, COLOR_GAIN_CSTR[1])
+            .ok()?,
+        settings
+            .get_float(STEAMVR_SECTION, COLOR_GAIN_CSTR[2])
+            .ok()?,
+    ];
+
+    // prevent user from turning everything black
+    let mut min = if current[0] + current[1] + current[2] < 0.11 {
+        0.1
+    } else {
+        0.0
+    };
+
+    match ch {
+        ColorChannel::R => {
+            settings
+                .set_float(
+                    STEAMVR_SECTION,
+                    COLOR_GAIN_CSTR[0],
+                    (current[0] + delta).clamp(min, 1.0),
+                )
+                .ok()?;
+        }
+        ColorChannel::G => {
+            settings
+                .set_float(
+                    STEAMVR_SECTION,
+                    COLOR_GAIN_CSTR[1],
+                    (current[1] + delta).clamp(min, 1.0),
+                )
+                .ok()?;
+        }
+        ColorChannel::B => {
+            settings
+                .set_float(
+                    STEAMVR_SECTION,
+                    COLOR_GAIN_CSTR[2],
+                    (current[2] + delta).clamp(min, 1.0),
+                )
+                .ok()?;
+        }
+        ColorChannel::All => {
+            min *= 0.3333;
+            settings
+                .set_float(
+                    STEAMVR_SECTION,
+                    COLOR_GAIN_CSTR[0],
+                    (current[0] + delta).clamp(min, 1.0),
+                )
+                .ok()?;
+            settings
+                .set_float(
+                    STEAMVR_SECTION,
+                    COLOR_GAIN_CSTR[1],
+                    (current[1] + delta).clamp(min, 1.0),
+                )
+                .ok()?;
+            settings
+                .set_float(
+                    STEAMVR_SECTION,
+                    COLOR_GAIN_CSTR[2],
+                    (current[2] + delta).clamp(min, 1.0),
+                )
+                .ok()?;
+        }
+    }
+
+    Some(())
+}

src/backend/openvr/input.rs

@@ -0,0 +1,338 @@
+use std::{array, fs::File, io::Write, time::Duration};
+
+use anyhow::bail;
+use ovr_overlay::{
+    input::{ActionHandle, ActionSetHandle, ActiveActionSet, InputManager, InputValueHandle},
+    sys::{
+        ETrackedControllerRole, ETrackedDeviceClass, ETrackedDeviceProperty,
+        ETrackingUniverseOrigin,
+    },
+    system::SystemManager,
+    TrackedDeviceIndex,
+};
+
+use crate::{
+    backend::input::{Haptics, TrackedDevice, TrackedDeviceRole},
+    config_io::CONFIG_ROOT_PATH,
+    state::AppState,
+};
+
+use super::helpers::{Affine3AConvert, OVRError};
+
+const SET_DEFAULT: &str = "/actions/default";
+const INPUT_SOURCES: [&str; 2] = ["/user/hand/left", "/user/hand/right"];
+const PATH_POSES: [&str; 2] = [
+    "/actions/default/in/LeftHand",
+    "/actions/default/in/RightHand",
+];
+const PATH_HAPTICS: [&str; 2] = [
+    "/actions/default/out/HapticsLeft",
+    "/actions/default/out/HapticsRight",
+];
+
+const PATH_CLICK: &str = "/actions/default/in/Click";
+const PATH_GRAB: &str = "/actions/default/in/Grab";
+const PATH_SCROLL: &str = "/actions/default/in/Scroll";
+const PATH_ALT_CLICK: &str = "/actions/default/in/AltClick";
+const PATH_SHOW_HIDE: &str = "/actions/default/in/ShowHide";
+const PATH_SPACE_DRAG: &str = "/actions/default/in/SpaceDrag";
+const PATH_SPACE_ROTATE: &str = "/actions/default/in/SpaceRotate";
+const PATH_CLICK_MODIFIER_RIGHT: &str = "/actions/default/in/ClickModifierRight";
+const PATH_CLICK_MODIFIER_MIDDLE: &str = "/actions/default/in/ClickModifierMiddle";
+
+const INPUT_ANY: InputValueHandle = InputValueHandle(ovr_overlay::sys::k_ulInvalidInputValueHandle);
+
+pub(super) struct OpenVrInputSource {
+    hands: [OpenVrHandSource; 2],
+    set_hnd: ActionSetHandle,
+    click_hnd: ActionHandle,
+    grab_hnd: ActionHandle,
+    scroll_hnd: ActionHandle,
+    alt_click_hnd: ActionHandle,
+    show_hide_hnd: ActionHandle,
+    space_drag_hnd: ActionHandle,
+    space_rotate_hnd: ActionHandle,
+    click_modifier_right_hnd: ActionHandle,
+    click_modifier_middle_hnd: ActionHandle,
+}
+
+pub(super) struct OpenVrHandSource {
+    has_pose: bool,
+    device: Option<TrackedDeviceIndex>,
+    input_hnd: InputValueHandle,
+    pose_hnd: ActionHandle,
+    haptics_hnd: ActionHandle,
+}
+
+impl OpenVrInputSource {
+    pub fn new(input: &mut InputManager) -> Result<Self, OVRError> {
+        let set_hnd = input.get_action_set_handle(SET_DEFAULT)?;
+
+        let click_hnd = input.get_action_handle(PATH_CLICK)?;
+        let grab_hnd = input.get_action_handle(PATH_GRAB)?;
+        let scroll_hnd = input.get_action_handle(PATH_SCROLL)?;
+        let alt_click_hnd = input.get_action_handle(PATH_ALT_CLICK)?;
+        let show_hide_hnd = input.get_action_handle(PATH_SHOW_HIDE)?;
+        let space_drag_hnd = input.get_action_handle(PATH_SPACE_DRAG)?;
+        let space_rotate_hnd = input.get_action_handle(PATH_SPACE_ROTATE)?;
+        let click_modifier_right_hnd = input.get_action_handle(PATH_CLICK_MODIFIER_RIGHT)?;
+        let click_modifier_middle_hnd = input.get_action_handle(PATH_CLICK_MODIFIER_MIDDLE)?;
+
+        let input_hnd: Vec<InputValueHandle> = INPUT_SOURCES
+            .iter()
+            .map(|path| Ok((input.get_input_source_handle(path))?))
+            .collect::<Result<_, OVRError>>()?;
+
+        let pose_hnd: Vec<ActionHandle> = PATH_POSES
+            .iter()
+            .map(|path| Ok((input.get_action_handle(path))?))
+            .collect::<Result<_, OVRError>>()?;
+
+        let haptics_hnd: Vec<ActionHandle> = PATH_HAPTICS
+            .iter()
+            .map(|path| Ok((input.get_action_handle(path))?))
+            .collect::<Result<_, OVRError>>()?;
+
+        let hands: [OpenVrHandSource; 2] = array::from_fn(|i| OpenVrHandSource {
+            has_pose: false,
+            device: None,
+            input_hnd: input_hnd[i],
+            pose_hnd: pose_hnd[i],
+            haptics_hnd: haptics_hnd[i],
+        });
+
+        Ok(OpenVrInputSource {
+            set_hnd,
+            click_hnd,
+            grab_hnd,
+            scroll_hnd,
+            alt_click_hnd,
+            show_hide_hnd,
+            space_drag_hnd,
+            space_rotate_hnd,
+            click_modifier_right_hnd,
+            click_modifier_middle_hnd,
+            hands,
+        })
+    }
+
+    pub fn haptics(&mut self, input: &mut InputManager, hand: usize, haptics: &Haptics) {
+        let hnd = self.hands[hand].haptics_hnd;
+        let _ = input.trigger_haptic_vibration_action(
+            hnd,
+            0.0,
+            Duration::from_secs_f32(haptics.duration),
+            haptics.frequency,
+            haptics.intensity,
+            INPUT_ANY,
+        );
+    }
+
+    pub fn update(
+        &mut self,
+        universe: ETrackingUniverseOrigin,
+        input: &mut InputManager,
+        system: &mut SystemManager,
+        app: &mut AppState,
+    ) {
+        let aas = ActiveActionSet(ovr_overlay::sys::VRActiveActionSet_t {
+            ulActionSet: self.set_hnd.0,
+            ulRestrictedToDevice: 0,
+            ulSecondaryActionSet: 0,
+            unPadding: 0,
+            nPriority: 0,
+        });
+
+        let _ = input.update_actions(&mut [aas]);
+
+        let devices = system.get_device_to_absolute_tracking_pose(universe.clone(), 0.005);
+        app.input_state.hmd = devices[0].mDeviceToAbsoluteTracking.to_affine();
+
+        for i in 0..2 {
+            let hand = &mut self.hands[i];
+            let app_hand = &mut app.input_state.pointers[i];
+
+            if let Some(device) = hand.device {
+                app_hand.raw_pose = devices[device.0 as usize]
+                    .mDeviceToAbsoluteTracking
+                    .to_affine();
+            }
+
+            hand.has_pose = false;
+
+            let _ = input
+                .get_pose_action_data_relative_to_now(
+                    hand.pose_hnd,
+                    universe.clone(),
+                    0.005,
+                    INPUT_ANY,
+                )
+                .map(|pose| {
+                    app_hand.pose = pose.0.pose.mDeviceToAbsoluteTracking.to_affine();
+                    hand.has_pose = true;
+                });
+
+            app_hand.now.click = input
+                .get_digital_action_data(self.click_hnd, hand.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            app_hand.now.grab = input
+                .get_digital_action_data(self.grab_hnd, hand.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            app_hand.now.alt_click = input
+                .get_digital_action_data(self.alt_click_hnd, hand.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            app_hand.now.show_hide = input
+                .get_digital_action_data(self.show_hide_hnd, hand.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            app_hand.now.space_drag = input
+                .get_digital_action_data(self.space_drag_hnd, hand.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            app_hand.now.space_rotate = input
+                .get_digital_action_data(self.space_rotate_hnd, hand.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            app_hand.now.click_modifier_right = input
+                .get_digital_action_data(self.click_modifier_right_hnd, hand.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            app_hand.now.click_modifier_middle = input
+                .get_digital_action_data(self.click_modifier_middle_hnd, hand.input_hnd)
+                .map(|x| x.0.bState)
+                .unwrap_or(false);
+
+            app_hand.now.scroll = input
+                .get_analog_action_data(self.scroll_hnd, hand.input_hnd)
+                .map(|x| x.0.y)
+                .unwrap_or(0.0);
+        }
+    }
+
+    pub fn update_devices(&mut self, system: &mut SystemManager, app: &mut AppState) {
+        app.input_state.devices.clear();
+        for idx in 0..TrackedDeviceIndex::MAX {
+            let device = TrackedDeviceIndex::new(idx as _).unwrap(); // safe
+            if !system.is_tracked_device_connected(device) {
+                continue;
+            }
+
+            let class = system.get_tracked_device_class(device);
+
+            let role = match class {
+                ETrackedDeviceClass::TrackedDeviceClass_HMD => TrackedDeviceRole::Hmd,
+                ETrackedDeviceClass::TrackedDeviceClass_Controller => {
+                    let role = system.get_controller_role_for_tracked_device_index(device);
+                    match role {
+                        ETrackedControllerRole::TrackedControllerRole_LeftHand => {
+                            self.hands[0].device = Some(device);
+                            TrackedDeviceRole::LeftHand
+                        }
+                        ETrackedControllerRole::TrackedControllerRole_RightHand => {
+                            self.hands[1].device = Some(device);
+                            TrackedDeviceRole::RightHand
+                        }
+                        _ => continue,
+                    }
+                }
+                ETrackedDeviceClass::TrackedDeviceClass_GenericTracker => {
+                    TrackedDeviceRole::Tracker
+                }
+                _ => continue,
+            };
+
+            if let Some(device) = get_tracked_device(system, device, role) {
+                app.input_state.devices.push(device);
+            }
+        }
+
+        app.input_state.devices.sort_by(|a, b| {
+            (a.soc.is_none() as u8)
+                .cmp(&(b.soc.is_none() as u8))
+                .then((a.role as u8).cmp(&(b.role as u8)))
+                .then(a.soc.unwrap_or(999.).total_cmp(&b.soc.unwrap_or(999.)))
+        });
+    }
+}
+
+fn get_tracked_device(
+    system: &mut SystemManager,
+    index: TrackedDeviceIndex,
+    role: TrackedDeviceRole,
+) -> Option<TrackedDevice> {
+    let soc = system
+        .get_tracked_device_property(
+            index,
+            ETrackedDeviceProperty::Prop_DeviceBatteryPercentage_Float,
+        )
+        .ok();
+
+    let charging = if soc.is_some() {
+        system
+            .get_tracked_device_property(index, ETrackedDeviceProperty::Prop_DeviceIsCharging_Bool)
+            .unwrap_or(false)
+    } else {
+        false
+    };
+
+    // TODO: cache this
+    let is_alvr = system
+        .get_tracked_device_property(
+            index,
+            ETrackedDeviceProperty::Prop_TrackingSystemName_String,
+        )
+        .map(|x: String| x.contains("ALVR"))
+        .unwrap_or(false);
+
+    if is_alvr {
+        // don't show ALVR's fake trackers on battery panel
+        return None;
+    }
+
+    Some(TrackedDevice {
+        valid: true,
+        index,
+        soc,
+        charging,
+        role,
+    })
+}
+
+pub fn set_action_manifest(input: &mut InputManager) -> anyhow::Result<()> {
+    let action_path = CONFIG_ROOT_PATH.join("actions.json");
+
+    File::create(&action_path)?.write_all(include_bytes!("../../res/actions.json"))?;
+
+    let binding_path = CONFIG_ROOT_PATH.join("actions_binding_knuckles.json");
+    if !binding_path.is_file() {
+        File::create(&binding_path)?
+            .write_all(include_bytes!("../../res/actions_binding_knuckles.json"))?;
+    }
+
+    let binding_path = CONFIG_ROOT_PATH.join("actions_binding_vive.json");
+    if !binding_path.is_file() {
+        File::create(&binding_path)?
+            .write_all(include_bytes!("../../res/actions_binding_vive.json"))?;
+    }
+
+    let binding_path = CONFIG_ROOT_PATH.join("actions_binding_oculus.json");
+    if !binding_path.is_file() {
+        File::create(&binding_path)?
+            .write_all(include_bytes!("../../res/actions_binding_oculus.json"))?;
+    }
+
+    if let Err(e) = input.set_action_manifest(action_path.as_path()) {
+        bail!("Failed to set action manifest: {}", e);
+    }
+    Ok(())
+}

src/backend/openvr/lines.rs

@@ -0,0 +1,192 @@
+use std::f32::consts::PI;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+
+use glam::{Affine3A, Vec3, Vec3A, Vec4};
+use idmap::IdMap;
+use ovr_overlay::overlay::OverlayManager;
+use ovr_overlay::sys::ETrackingUniverseOrigin;
+use vulkano::command_buffer::CommandBufferUsage;
+use vulkano::format::Format;
+use vulkano::image::view::ImageView;
+use vulkano::image::ImageLayout;
+
+use crate::backend::overlay::{OverlayData, OverlayRenderer, OverlayState, SplitOverlayBackend};
+use crate::graphics::WlxGraphics;
+use crate::state::AppState;
+
+use super::overlay::OpenVrOverlayData;
+
+static AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(1);
+
+pub(super) struct LinePool {
+    lines: IdMap<usize, OverlayData<OpenVrOverlayData>>,
+    view: Arc<ImageView>,
+    colors: [Vec4; 5],
+}
+
+impl LinePool {
+    pub fn new(graphics: Arc<WlxGraphics>) -> anyhow::Result<Self> {
+        let mut command_buffer =
+            graphics.create_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+
+        let buf = vec![255; 16];
+
+        let texture = command_buffer.texture2d(2, 2, Format::R8G8B8A8_UNORM, &buf)?;
+        command_buffer.build_and_execute_now()?;
+
+        graphics
+            .transition_layout(
+                texture.clone(),
+                ImageLayout::ShaderReadOnlyOptimal,
+                ImageLayout::TransferSrcOptimal,
+            )?
+            .wait(None)?;
+
+        let view = ImageView::new_default(texture)?;
+
+        Ok(LinePool {
+            lines: IdMap::new(),
+            view,
+            colors: [
+                Vec4::new(1., 1., 1., 1.),
+                Vec4::new(0., 0.375, 0.5, 1.),
+                Vec4::new(0.69, 0.188, 0., 1.),
+                Vec4::new(0.375, 0., 0.5, 1.),
+                Vec4::new(1., 0., 0., 1.),
+            ],
+        })
+    }
+
+    pub fn allocate(&mut self) -> usize {
+        let id = AUTO_INCREMENT.fetch_add(1, Ordering::Relaxed);
+
+        let mut data = OverlayData::<OpenVrOverlayData> {
+            state: OverlayState {
+                name: Arc::from(format!("wlx-line{}", id)),
+                show_hide: true,
+                ..Default::default()
+            },
+            backend: Box::new(SplitOverlayBackend {
+                renderer: Box::new(StaticRenderer {
+                    view: self.view.clone(),
+                }),
+                ..Default::default()
+            }),
+            data: OpenVrOverlayData {
+                width: 0.002,
+                override_width: true,
+                ..Default::default()
+            },
+            ..Default::default()
+        };
+        data.data.sort_order = 69;
+        data.state.dirty = true;
+
+        self.lines.insert(id, data);
+        id
+    }
+
+    pub fn draw_from(
+        &mut self,
+        id: usize,
+        mut from: Affine3A,
+        len: f32,
+        color: usize,
+        hmd: &Affine3A,
+    ) {
+        let rotation = Affine3A::from_axis_angle(Vec3::X, -PI * 0.5);
+
+        from.translation += from.transform_vector3a(Vec3A::NEG_Z) * (len * 0.5);
+        let mut transform = from * rotation * Affine3A::from_scale(Vec3::new(1., len / 0.002, 1.));
+
+        let to_hmd = hmd.translation - from.translation;
+        let sides = [Vec3A::Z, Vec3A::X, Vec3A::NEG_Z, Vec3A::NEG_X];
+        let rotations = [
+            Affine3A::IDENTITY,
+            Affine3A::from_axis_angle(Vec3::Y, PI * 0.5),
+            Affine3A::from_axis_angle(Vec3::Y, PI * 1.0),
+            Affine3A::from_axis_angle(Vec3::Y, PI * 1.5),
+        ];
+        let mut closest = (0, 0.0);
+        for (i, &side) in sides.iter().enumerate() {
+            let dot = to_hmd.dot(transform.transform_vector3a(side));
+            if i == 0 || dot > closest.1 {
+                closest = (i, dot);
+            }
+        }
+
+        transform *= rotations[closest.0];
+
+        debug_assert!(color < self.colors.len());
+
+        self.draw_transform(id, transform, self.colors[color]);
+    }
+
+    fn draw_transform(&mut self, id: usize, transform: Affine3A, color: Vec4) {
+        if let Some(data) = self.lines.get_mut(id) {
+            data.state.want_visible = true;
+            data.state.transform = transform;
+            data.data.color = color;
+        } else {
+            log::warn!("Line {} does not exist", id);
+        }
+    }
+
+    pub fn hide(&mut self, id: usize) {
+        if let Some(data) = self.lines.get_mut(id) {
+            data.state.want_visible = false;
+        } else {
+            log::warn!("Line {} does not exist", id);
+        }
+    }
+
+    pub fn update(
+        &mut self,
+        universe: ETrackingUniverseOrigin,
+        overlay: &mut OverlayManager,
+        app: &mut AppState,
+    ) -> anyhow::Result<()> {
+        for data in self.lines.values_mut() {
+            data.after_input(overlay, app)?;
+            if data.state.want_visible {
+                if data.state.dirty {
+                    data.upload_texture(overlay, &app.graphics);
+                    data.state.dirty = false;
+                }
+
+                data.upload_transform(universe.clone(), overlay);
+                data.upload_color(overlay);
+            }
+        }
+        Ok(())
+    }
+
+    pub fn mark_dirty(&mut self) {
+        for data in self.lines.values_mut() {
+            data.state.dirty = true;
+        }
+    }
+}
+
+struct StaticRenderer {
+    view: Arc<ImageView>,
+}
+
+impl OverlayRenderer for StaticRenderer {
+    fn init(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+    fn pause(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+    fn resume(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+    fn render(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+    fn view(&mut self) -> Option<Arc<ImageView>> {
+        Some(self.view.clone())
+    }
+}

src/backend/openvr/manifest.rs

@@ -0,0 +1,88 @@
+use std::{fs::File, io::Read};
+
+use anyhow::bail;
+use json::{array, object};
+use ovr_overlay::applications::ApplicationsManager;
+
+use crate::config_io::CONFIG_ROOT_PATH;
+
+const APP_KEY: &str = "galister.wlxoverlay-s";
+
+pub(super) fn install_manifest(app_mgr: &mut ApplicationsManager) -> anyhow::Result<()> {
+    let manifest_path = CONFIG_ROOT_PATH.join("wlx-overlay-s.vrmanifest");
+
+    let appimage_path = std::env::var("APPIMAGE");
+    let executable_pathbuf = std::env::current_exe()?;
+
+    let executable_path = match appimage_path {
+        Ok(ref path) => path,
+        Err(_) => executable_pathbuf
+            .to_str()
+            .ok_or_else(|| anyhow::anyhow!("Invalid executable path"))?,
+    };
+
+    if let Ok(true) = app_mgr.is_application_installed(APP_KEY) {
+        if let Ok(mut file) = File::open(&manifest_path) {
+            let mut buf = String::new();
+            if file.read_to_string(&mut buf).is_ok() {
+                let manifest: json::JsonValue = json::parse(&buf)?;
+                if manifest["applications"][0]["binary_path_linux"] == executable_path {
+                    log::info!("Manifest already up to date");
+                    return Ok(());
+                }
+            }
+        }
+    }
+
+    let manifest = object! {
+        source: "builtin",
+        applications: array![
+            object! {
+                app_key: APP_KEY,
+                launch_type: "binary",
+                binary_path_linux: executable_path,
+                is_dashboard_overlay: true,
+                strings: object!{
+                    "en_us": object!{
+                        name: "WlxOverlay-S",
+                        description: "A lightweight Wayland desktop overlay for OpenVR/OpenXR",
+                    },
+                },
+            },
+        ],
+    };
+
+    let Ok(mut file) = File::create(&manifest_path) else {
+        bail!("Failed to create manifest file at {:?}", manifest_path);
+    };
+
+    if let Err(e) = manifest.write(&mut file) {
+        bail!(
+            "Failed to write manifest file at {:?}: {:?}",
+            manifest_path,
+            e
+        );
+    };
+
+    if let Err(e) = app_mgr.add_application_manifest(&manifest_path, false) {
+        bail!("Failed to add manifest to OpenVR: {}", e.description());
+    };
+
+    if let Err(e) = app_mgr.set_application_auto_launch(APP_KEY, true) {
+        bail!("Failed to set auto launch: {}", e.description());
+    };
+
+    Ok(())
+}
+
+pub(super) fn uninstall_manifest(app_mgr: &mut ApplicationsManager) -> anyhow::Result<()> {
+    let manifest_path = CONFIG_ROOT_PATH.join("wlx-overlay-s.vrmanifest");
+
+    if let Ok(true) = app_mgr.is_application_installed(APP_KEY) {
+        if let Err(e) = app_mgr.remove_application_manifest(&manifest_path) {
+            bail!("Failed to remove manifest from OpenVR: {}", e.description());
+        };
+        log::info!("Uninstalled manifest");
+    }
+    Ok(())
+}

src/backend/openvr/mod.rs

@@ -0,0 +1,301 @@
+use std::{
+    collections::VecDeque,
+    sync::{
+        atomic::{AtomicBool, AtomicUsize, Ordering},
+        Arc,
+    },
+    time::{Duration, Instant},
+};
+
+use anyhow::{anyhow, Result};
+use ovr_overlay::{
+    sys::{ETrackedDeviceProperty, EVRApplicationType, EVREventType},
+    TrackedDeviceIndex,
+};
+use vulkano::{
+    device::{physical::PhysicalDevice, DeviceExtensions},
+    instance::InstanceExtensions,
+    Handle, VulkanObject,
+};
+
+use crate::{
+    backend::{
+        common::SystemTask,
+        input::interact,
+        notifications::NotificationManager,
+        openvr::{
+            helpers::adjust_gain,
+            input::{set_action_manifest, OpenVrInputSource},
+            lines::LinePool,
+            manifest::{install_manifest, uninstall_manifest},
+            overlay::OpenVrOverlayData,
+        },
+        overlay::OverlayData,
+    },
+    graphics::WlxGraphics,
+    overlays::watch::{watch_fade, WATCH_NAME},
+    state::AppState,
+};
+
+use super::common::{BackendError, OverlayContainer, TaskType};
+
+pub mod helpers;
+pub mod input;
+pub mod lines;
+pub mod manifest;
+pub mod overlay;
+pub mod playspace;
+
+static FRAME_COUNTER: AtomicUsize = AtomicUsize::new(0);
+
+pub fn openvr_uninstall() {
+    let app_type = EVRApplicationType::VRApplication_Overlay;
+    let Ok(context) = ovr_overlay::Context::init(app_type) else {
+        log::error!("Uninstall failed: could not reach OpenVR");
+        return;
+    };
+
+    let mut app_mgr = context.applications_mngr();
+    let _ = uninstall_manifest(&mut app_mgr);
+}
+
+pub fn openvr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
+    let app_type = EVRApplicationType::VRApplication_Overlay;
+    let Ok(context) = ovr_overlay::Context::init(app_type) else {
+        log::warn!("Will not use OpenVR: Context init failed");
+        return Err(BackendError::NotSupported);
+    };
+
+    log::info!("Using OpenVR runtime");
+
+    let mut app_mgr = context.applications_mngr();
+    let mut input_mgr = context.input_mngr();
+    let mut system_mgr = context.system_mngr();
+    let mut overlay_mgr = context.overlay_mngr();
+    let mut settings_mgr = context.settings_mngr();
+    let mut chaperone_mgr = context.chaperone_setup_mngr();
+    let mut compositor_mgr = context.compositor_mngr();
+
+    let device_extensions_fn = |device: &PhysicalDevice| {
+        let names = compositor_mgr.get_vulkan_device_extensions_required(device.handle().as_raw());
+        let ext = DeviceExtensions::from_iter(names.iter().map(|s| s.as_str()));
+        ext
+    };
+
+    let mut compositor_mngr = context.compositor_mngr();
+    let instance_extensions = {
+        let names = compositor_mngr.get_vulkan_instance_extensions_required();
+        InstanceExtensions::from_iter(names.iter().map(|s| s.as_str()))
+    };
+
+    let mut state = {
+        let graphics = WlxGraphics::new_openvr(instance_extensions, device_extensions_fn)?;
+        AppState::from_graphics(graphics)?
+    };
+
+    let _ = install_manifest(&mut app_mgr);
+
+    let mut overlays = OverlayContainer::<OpenVrOverlayData>::new(&mut state)?;
+    let mut notifications = NotificationManager::new();
+    notifications.run_dbus();
+    notifications.run_udp();
+
+    let mut playspace = playspace::PlayspaceMover::new();
+    playspace.playspace_changed(&mut compositor_mgr, &mut chaperone_mgr);
+
+    #[cfg(feature = "osc")]
+    let mut osc_sender =
+        crate::backend::osc::OscSender::new(state.session.config.osc_out_port).ok();
+
+    state.hid_provider.set_desktop_extent(overlays.extent);
+
+    set_action_manifest(&mut input_mgr)?;
+
+    let mut input_source = OpenVrInputSource::new(&mut input_mgr)?;
+
+    let Ok(refresh_rate) = system_mgr.get_tracked_device_property::<f32>(
+        TrackedDeviceIndex::HMD,
+        ETrackedDeviceProperty::Prop_DisplayFrequency_Float,
+    ) else {
+        return Err(BackendError::Fatal(anyhow!(
+            "Failed to get HMD refresh rate"
+        )));
+    };
+
+    log::info!("HMD running @ {} Hz", refresh_rate);
+
+    let watch_id = overlays.get_by_name(WATCH_NAME).unwrap().state.id; // want panic
+
+    // want at least half refresh rate
+    let frame_timeout = 2 * (1000.0 / refresh_rate).floor() as u32;
+
+    let mut next_device_update = Instant::now();
+    let mut due_tasks = VecDeque::with_capacity(4);
+
+    let mut lines = LinePool::new(state.graphics.clone())?;
+    let pointer_lines = [lines.allocate(), lines.allocate()];
+
+    'main_loop: loop {
+        let _ = overlay_mgr.wait_frame_sync(frame_timeout);
+
+        if !running.load(Ordering::Relaxed) {
+            log::warn!("Received shutdown signal.");
+            break 'main_loop;
+        }
+
+        let cur_frame = FRAME_COUNTER.fetch_add(1, Ordering::Relaxed);
+
+        while let Some(event) = system_mgr.poll_next_event() {
+            match event.event_type {
+                EVREventType::VREvent_Quit => {
+                    log::warn!("Received quit event, shutting down.");
+                    break 'main_loop;
+                }
+                EVREventType::VREvent_TrackedDeviceActivated
+                | EVREventType::VREvent_TrackedDeviceDeactivated
+                | EVREventType::VREvent_TrackedDeviceUpdated => {
+                    next_device_update = Instant::now();
+                }
+                EVREventType::VREvent_SeatedZeroPoseReset
+                | EVREventType::VREvent_StandingZeroPoseReset
+                | EVREventType::VREvent_ChaperoneUniverseHasChanged
+                | EVREventType::VREvent_SceneApplicationChanged => {
+                    playspace.playspace_changed(&mut compositor_mgr, &mut chaperone_mgr);
+                }
+                _ => {}
+            }
+        }
+
+        if next_device_update <= Instant::now() {
+            input_source.update_devices(&mut system_mgr, &mut state);
+            next_device_update = Instant::now() + Duration::from_secs(30);
+        }
+
+        notifications.submit_pending(&mut state);
+
+        state.tasks.retrieve_due(&mut due_tasks);
+        while let Some(task) = due_tasks.pop_front() {
+            match task {
+                TaskType::Global(f) => f(&mut state),
+                TaskType::Overlay(sel, f) => {
+                    if let Some(o) = overlays.mut_by_selector(&sel) {
+                        f(&mut state, &mut o.state);
+                    }
+                }
+                TaskType::CreateOverlay(sel, f) => {
+                    let None = overlays.mut_by_selector(&sel) else {
+                        continue;
+                    };
+
+                    let Some((mut state, backend)) = f(&mut state) else {
+                        continue;
+                    };
+                    state.birthframe = cur_frame;
+
+                    overlays.add(OverlayData {
+                        state,
+                        backend,
+                        ..Default::default()
+                    });
+                }
+                TaskType::DropOverlay(sel) => {
+                    if let Some(o) = overlays.mut_by_selector(&sel) {
+                        if o.state.birthframe < cur_frame {
+                            o.destroy(&mut overlay_mgr);
+                            overlays.remove_by_selector(&sel);
+                        }
+                    }
+                }
+                TaskType::System(task) => match task {
+                    SystemTask::ColorGain(channel, value) => {
+                        let _ = adjust_gain(&mut settings_mgr, channel, value);
+                    }
+                    SystemTask::FixFloor => {
+                        playspace.fix_floor(&mut chaperone_mgr, &state.input_state);
+                    }
+                    SystemTask::ResetPlayspace => {
+                        playspace.reset_offset(&mut chaperone_mgr, &state.input_state);
+                    }
+                },
+            }
+        }
+
+        let universe = playspace.get_universe();
+
+        state.input_state.pre_update();
+        input_source.update(
+            universe.clone(),
+            &mut input_mgr,
+            &mut system_mgr,
+            &mut state,
+        );
+        state.input_state.post_update();
+
+        if state
+            .input_state
+            .pointers
+            .iter()
+            .any(|p| p.now.show_hide && !p.before.show_hide)
+        {
+            lines.mark_dirty(); // workaround to prevent lines from not showing
+            overlays.show_hide(&mut state);
+        }
+
+        overlays
+            .iter_mut()
+            .for_each(|o| o.state.auto_movement(&mut state));
+
+        watch_fade(&mut state, overlays.mut_by_id(watch_id).unwrap()); // want panic
+        playspace.update(&mut chaperone_mgr, &mut overlays, &state);
+
+        let lengths_haptics = interact(&mut overlays, &mut state);
+        for (idx, (len, haptics)) in lengths_haptics.iter().enumerate() {
+            lines.draw_from(
+                pointer_lines[idx],
+                state.input_state.pointers[idx].pose,
+                *len,
+                state.input_state.pointers[idx].interaction.mode as usize + 1,
+                &state.input_state.hmd,
+            );
+            if let Some(haptics) = haptics {
+                input_source.haptics(&mut input_mgr, idx, haptics)
+            }
+        }
+
+        lines.update(universe.clone(), &mut overlay_mgr, &mut state)?;
+
+        for o in overlays.iter_mut() {
+            o.after_input(&mut overlay_mgr, &mut state)?;
+        }
+
+        #[cfg(feature = "osc")]
+        if let Some(ref mut sender) = osc_sender {
+            let _ = sender.send_params(&overlays);
+        };
+
+        log::trace!("Rendering frame");
+
+        for o in overlays.iter_mut() {
+            if o.state.want_visible {
+                o.render(&mut state)?;
+            }
+        }
+
+        log::trace!("Rendering overlays");
+
+        overlays
+            .iter_mut()
+            .for_each(|o| o.after_render(universe.clone(), &mut overlay_mgr, &state.graphics));
+
+        // chaperone
+
+        // close font handles?
+
+        state.hid_provider.on_new_frame();
+    }
+
+    log::warn!("OpenVR shutdown");
+    // context.shutdown() called by Drop
+
+    Ok(())
+}

src/backend/openvr/overlay.rs

@@ -0,0 +1,284 @@
+use core::f32;
+
+use glam::Vec4;
+use ovr_overlay::{
+    overlay::{OverlayHandle, OverlayManager},
+    pose::Matrix3x4,
+    sys::{ETrackingUniverseOrigin, VRVulkanTextureData_t},
+};
+use vulkano::{Handle, VulkanObject};
+
+use crate::{
+    backend::overlay::{OverlayData, RelativeTo},
+    graphics::WlxGraphics,
+    overlays::watch::WATCH_NAME,
+    state::AppState,
+};
+
+use super::helpers::Affine3AConvert;
+
+#[derive(Default)]
+pub(super) struct OpenVrOverlayData {
+    pub(super) handle: Option<OverlayHandle>,
+    pub(super) last_image: Option<u64>,
+    pub(super) visible: bool,
+    pub(super) color: Vec4,
+    pub(super) sort_order: u32,
+    pub(crate) width: f32,
+    pub(super) override_width: bool,
+    pub(super) relative_to: RelativeTo,
+}
+
+impl OverlayData<OpenVrOverlayData> {
+    pub(super) fn initialize(
+        &mut self,
+        overlay: &mut OverlayManager,
+        app: &mut AppState,
+    ) -> anyhow::Result<OverlayHandle> {
+        let key = format!("wlx-{}", self.state.name);
+        log::debug!("Create overlay with key: {}", &key);
+        let handle = match overlay.create_overlay(&key, &key) {
+            Ok(handle) => handle,
+            Err(e) => {
+                panic!("Failed to create overlay: {}", e);
+            }
+        };
+        log::debug!("{}: initialize", self.state.name);
+
+        //watch
+        if *self.state.name == *WATCH_NAME {
+            self.data.sort_order = 68;
+        }
+
+        self.data.handle = Some(handle);
+        self.data.color = Vec4::ONE;
+
+        self.init(app)?;
+
+        if self.data.width < f32::EPSILON {
+            self.data.width = 1.0;
+        }
+
+        self.upload_width(overlay);
+        self.upload_color(overlay);
+        self.upload_alpha(overlay);
+        self.upload_curvature(overlay);
+        self.upload_sort_order(overlay);
+
+        Ok(handle)
+    }
+
+    pub(super) fn after_input(
+        &mut self,
+        overlay: &mut OverlayManager,
+        app: &mut AppState,
+    ) -> anyhow::Result<()> {
+        if self.state.want_visible && !self.data.visible {
+            self.show_internal(overlay, app)?;
+        } else if !self.state.want_visible && self.data.visible {
+            self.hide_internal(overlay, app)?;
+        }
+        Ok(())
+    }
+
+    pub(super) fn after_render(
+        &mut self,
+        universe: ETrackingUniverseOrigin,
+        overlay: &mut OverlayManager,
+        graphics: &WlxGraphics,
+    ) {
+        if self.data.visible {
+            if self.state.dirty {
+                self.upload_curvature(overlay);
+
+                self.upload_transform(universe, overlay);
+                self.upload_alpha(overlay);
+                self.state.dirty = false;
+            }
+            self.upload_texture(overlay, graphics);
+        }
+    }
+
+    fn show_internal(
+        &mut self,
+        overlay: &mut OverlayManager,
+        app: &mut AppState,
+    ) -> anyhow::Result<()> {
+        let handle = match self.data.handle {
+            Some(handle) => handle,
+            None => self.initialize(overlay, app)?,
+        };
+        log::debug!("{}: show", self.state.name);
+        if let Err(e) = overlay.set_visibility(handle, true) {
+            log::error!("{}: Failed to show overlay: {}", self.state.name, e);
+        }
+        self.data.visible = true;
+        self.backend.resume(app)
+    }
+
+    fn hide_internal(
+        &mut self,
+        overlay: &mut OverlayManager,
+        app: &mut AppState,
+    ) -> anyhow::Result<()> {
+        let Some(handle) = self.data.handle else {
+            return Ok(());
+        };
+        log::debug!("{}: hide", self.state.name);
+        if let Err(e) = overlay.set_visibility(handle, false) {
+            log::error!("{}: Failed to hide overlay: {}", self.state.name, e);
+        }
+        self.data.visible = false;
+        self.backend.pause(app)
+    }
+
+    pub(super) fn upload_alpha(&self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+        if let Err(e) = overlay.set_opacity(handle, self.state.alpha) {
+            log::error!("{}: Failed to set overlay alpha: {}", self.state.name, e);
+        }
+    }
+
+    pub(super) fn upload_color(&self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+        if let Err(e) = overlay.set_tint(
+            handle,
+            ovr_overlay::ColorTint {
+                r: self.data.color.x,
+                g: self.data.color.y,
+                b: self.data.color.z,
+                a: self.data.color.w,
+            },
+        ) {
+            log::error!("{}: Failed to set overlay tint: {}", self.state.name, e);
+        }
+    }
+
+    fn upload_width(&self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+        if let Err(e) = overlay.set_width(handle, self.data.width) {
+            log::error!("{}: Failed to set overlay width: {}", self.state.name, e);
+        }
+    }
+
+    fn upload_curvature(&self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+        if let Err(e) = overlay.set_curvature(handle, self.state.curvature.unwrap_or(0.0)) {
+            log::error!(
+                "{}: Failed to set overlay curvature: {}",
+                self.state.name,
+                e
+            );
+        }
+    }
+
+    fn upload_sort_order(&self, overlay: &mut OverlayManager) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+        if let Err(e) = overlay.set_sort_order(handle, self.data.sort_order) {
+            log::error!("{}: Failed to set overlay z order: {}", self.state.name, e);
+        }
+    }
+
+    pub(super) fn upload_transform(
+        &self,
+        universe: ETrackingUniverseOrigin,
+        overlay: &mut OverlayManager,
+    ) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+
+        let transform = Matrix3x4::from_affine(&self.state.transform);
+
+        if let Err(e) = overlay.set_transform_absolute(handle, universe, &transform) {
+            log::error!(
+                "{}: Failed to set overlay transform: {}",
+                self.state.name,
+                e
+            );
+        }
+    }
+
+    pub(super) fn upload_texture(&mut self, overlay: &mut OverlayManager, graphics: &WlxGraphics) {
+        let Some(handle) = self.data.handle else {
+            log::debug!("{}: No overlay handle", self.state.name);
+            return;
+        };
+
+        let Some(view) = self.backend.view() else {
+            log::debug!("{}: Not rendered", self.state.name);
+            return;
+        };
+
+        let image = view.image().clone();
+
+        let raw_image = image.handle().as_raw();
+
+        if let Some(last_image) = self.data.last_image {
+            if last_image == raw_image {
+                return;
+            }
+        }
+
+        let dimensions = image.extent();
+        if !self.data.override_width {
+            let new_width = ((dimensions[0] as f32) / (dimensions[1] as f32)).min(1.0);
+            if (new_width - self.data.width).abs() > f32::EPSILON {
+                log::info!("{}: New width {}", self.state.name, new_width);
+                self.data.width = new_width;
+                self.upload_width(overlay);
+            }
+        }
+
+        let format = image.format();
+
+        let mut texture = VRVulkanTextureData_t {
+            m_nImage: raw_image,
+            m_nFormat: format as _,
+            m_nWidth: dimensions[0],
+            m_nHeight: dimensions[1],
+            m_nSampleCount: image.samples() as u32,
+            m_pDevice: graphics.device.handle().as_raw() as *mut _,
+            m_pPhysicalDevice: graphics.device.physical_device().handle().as_raw() as *mut _,
+            m_pInstance: graphics.instance.handle().as_raw() as *mut _,
+            m_pQueue: graphics.queue.handle().as_raw() as *mut _,
+            m_nQueueFamilyIndex: graphics.queue.queue_family_index(),
+        };
+        log::debug!(
+            "{}: UploadTex {:?}, {}x{}, {:?}",
+            self.state.name,
+            format,
+            texture.m_nWidth,
+            texture.m_nHeight,
+            image.usage()
+        );
+        if let Err(e) = overlay.set_image_vulkan(handle, &mut texture) {
+            log::error!("{}: Failed to set overlay texture: {}", self.state.name, e);
+        }
+    }
+
+    pub(super) fn destroy(&mut self, overlay: &mut OverlayManager) {
+        if let Some(handle) = self.data.handle {
+            log::debug!("{}: destroy", self.state.name);
+            if let Err(e) = overlay.destroy_overlay(handle) {
+                log::error!("{}: Failed to destroy overlay: {}", self.state.name, e);
+            }
+        }
+    }
+}

src/backend/openvr/playspace.rs

@@ -0,0 +1,291 @@
+use glam::{Affine3A, Quat, Vec3, Vec3A};
+use ovr_overlay::{
+    chaperone_setup::ChaperoneSetupManager,
+    compositor::CompositorManager,
+    sys::{EChaperoneConfigFile, ETrackingUniverseOrigin, HmdMatrix34_t},
+};
+
+use crate::{
+    backend::{common::OverlayContainer, input::InputState},
+    state::AppState,
+};
+
+use super::{helpers::Affine3AConvert, overlay::OpenVrOverlayData};
+
+struct MoverData<T> {
+    pose: Affine3A,
+    hand: usize,
+    hand_pose: T,
+}
+
+pub(super) struct PlayspaceMover {
+    universe: ETrackingUniverseOrigin,
+    drag: Option<MoverData<Vec3A>>,
+    rotate: Option<MoverData<Quat>>,
+}
+
+impl PlayspaceMover {
+    pub fn new() -> Self {
+        Self {
+            universe: ETrackingUniverseOrigin::TrackingUniverseRawAndUncalibrated,
+            drag: None,
+            rotate: None,
+        }
+    }
+
+    pub fn update(
+        &mut self,
+        chaperone_mgr: &mut ChaperoneSetupManager,
+        overlays: &mut OverlayContainer<OpenVrOverlayData>,
+        state: &AppState,
+    ) {
+        let universe = self.universe.clone();
+
+        if let Some(data) = self.rotate.as_mut() {
+            let pointer = &state.input_state.pointers[data.hand];
+            if !pointer.now.space_rotate {
+                self.rotate = None;
+                log::info!("End space rotate");
+                return;
+            }
+
+            let new_hand =
+                Quat::from_affine3(&(data.pose * state.input_state.pointers[data.hand].raw_pose));
+
+            let dq = new_hand * data.hand_pose.conjugate();
+            let rel_y = f32::atan2(
+                2.0 * (dq.y * dq.w + dq.x * dq.z),
+                (2.0 * (dq.w * dq.w + dq.x * dq.x)) - 1.0,
+            );
+
+            let mut space_transform = Affine3A::from_rotation_y(rel_y);
+            let offset = (space_transform.transform_vector3a(state.input_state.hmd.translation)
+                - state.input_state.hmd.translation)
+                * -1.0;
+            let mut overlay_transform = Affine3A::from_rotation_y(-rel_y);
+
+            overlay_transform.translation = offset;
+            space_transform.translation = offset;
+
+            overlays.iter_mut().for_each(|overlay| {
+                if overlay.state.grabbable {
+                    overlay.state.dirty = true;
+                    overlay.state.transform.translation =
+                        overlay_transform.transform_point3a(overlay.state.transform.translation);
+                }
+            });
+
+            data.pose *= space_transform;
+            data.hand_pose = new_hand;
+
+            if self.universe == ETrackingUniverseOrigin::TrackingUniverseStanding {
+                apply_chaperone_transform(space_transform.inverse(), chaperone_mgr);
+            }
+            set_working_copy(&universe, chaperone_mgr, &data.pose);
+            chaperone_mgr.commit_working_copy(EChaperoneConfigFile::EChaperoneConfigFile_Live);
+        } else {
+            for (i, pointer) in state.input_state.pointers.iter().enumerate() {
+                if pointer.now.space_rotate {
+                    let Some(mat) = get_working_copy(&universe, chaperone_mgr) else {
+                        log::warn!("Can't space rotate - failed to get zero pose");
+                        return;
+                    };
+                    let hand_pose = Quat::from_affine3(&(mat * pointer.raw_pose));
+                    self.rotate = Some(MoverData {
+                        pose: mat,
+                        hand: i,
+                        hand_pose,
+                    });
+                    self.drag = None;
+                    log::info!("Start space rotate");
+                    return;
+                }
+            }
+        }
+
+        if let Some(data) = self.drag.as_mut() {
+            let pointer = &state.input_state.pointers[data.hand];
+            if !pointer.now.space_drag {
+                self.drag = None;
+                log::info!("End space drag");
+                return;
+            }
+
+            let new_hand = data
+                .pose
+                .transform_point3a(state.input_state.pointers[data.hand].raw_pose.translation);
+            let relative_pos = new_hand - data.hand_pose;
+
+            if relative_pos.length_squared() > 1000.0 {
+                log::warn!("Space drag too fast, ignoring");
+                return;
+            }
+
+            let overlay_offset = data.pose.inverse().transform_vector3a(relative_pos) * -1.0;
+
+            overlays.iter_mut().for_each(|overlay| {
+                if overlay.state.grabbable {
+                    overlay.state.dirty = true;
+                    overlay.state.transform.translation += overlay_offset;
+                }
+            });
+
+            data.pose.translation += relative_pos;
+            data.hand_pose = new_hand;
+
+            if self.universe == ETrackingUniverseOrigin::TrackingUniverseStanding {
+                apply_chaperone_offset(overlay_offset, chaperone_mgr);
+            }
+            set_working_copy(&universe, chaperone_mgr, &data.pose);
+            chaperone_mgr.commit_working_copy(EChaperoneConfigFile::EChaperoneConfigFile_Live);
+        } else {
+            for (i, pointer) in state.input_state.pointers.iter().enumerate() {
+                if pointer.now.space_drag {
+                    let Some(mat) = get_working_copy(&universe, chaperone_mgr) else {
+                        log::warn!("Can't space drag - failed to get zero pose");
+                        return;
+                    };
+                    let hand_pos = mat.transform_point3a(pointer.raw_pose.translation);
+                    self.drag = Some(MoverData {
+                        pose: mat,
+                        hand: i,
+                        hand_pose: hand_pos,
+                    });
+                    self.rotate = None;
+                    log::info!("Start space drag");
+                    return;
+                }
+            }
+        }
+    }
+
+    pub fn reset_offset(&mut self, chaperone_mgr: &mut ChaperoneSetupManager, input: &InputState) {
+        let mut height = 1.6;
+        if let Some(mat) = get_working_copy(&self.universe, chaperone_mgr) {
+            height = input.hmd.translation.y - mat.translation.y;
+            if self.universe == ETrackingUniverseOrigin::TrackingUniverseStanding {
+                apply_chaperone_transform(mat, chaperone_mgr);
+            }
+        }
+
+        let xform = if self.universe == ETrackingUniverseOrigin::TrackingUniverseSeated {
+            Affine3A::from_translation(Vec3::NEG_Y * height)
+        } else {
+            Affine3A::IDENTITY
+        };
+
+        set_working_copy(&self.universe, chaperone_mgr, &xform);
+        chaperone_mgr.commit_working_copy(EChaperoneConfigFile::EChaperoneConfigFile_Live);
+
+        if self.drag.is_some() {
+            log::info!("Space drag interrupted by manual reset");
+            self.drag = None;
+        }
+        if self.rotate.is_some() {
+            log::info!("Space rotate interrupted by manual reset");
+            self.rotate = None;
+        }
+    }
+
+    pub fn fix_floor(&mut self, chaperone_mgr: &mut ChaperoneSetupManager, input: &InputState) {
+        let y1 = input.pointers[0].pose.translation.y;
+        let y2 = input.pointers[1].pose.translation.y;
+        let Some(mut mat) = get_working_copy(&self.universe, chaperone_mgr) else {
+            log::warn!("Can't fix floor - failed to get zero pose");
+            return;
+        };
+        let offset = y1.min(y2) - 0.03;
+        mat.translation.y += offset;
+
+        set_working_copy(&self.universe, chaperone_mgr, &mat);
+        chaperone_mgr.commit_working_copy(EChaperoneConfigFile::EChaperoneConfigFile_Live);
+    }
+
+    pub fn playspace_changed(
+        &mut self,
+        compositor_mgr: &mut CompositorManager,
+        _chaperone_mgr: &mut ChaperoneSetupManager,
+    ) {
+        let new_universe = compositor_mgr.get_tracking_space();
+        if new_universe != self.universe {
+            log::info!(
+                "Playspace changed: {} -> {}",
+                universe_str(&self.universe),
+                universe_str(&new_universe)
+            );
+            self.universe = new_universe;
+        }
+
+        if self.drag.is_some() {
+            log::info!("Space drag interrupted by external change");
+            self.drag = None;
+        }
+        if self.rotate.is_some() {
+            log::info!("Space rotate interrupted by external change");
+            self.rotate = None;
+        }
+    }
+
+    pub fn get_universe(&self) -> ETrackingUniverseOrigin {
+        self.universe.clone()
+    }
+}
+
+fn universe_str(universe: &ETrackingUniverseOrigin) -> &'static str {
+    match universe {
+        ETrackingUniverseOrigin::TrackingUniverseSeated => "Seated",
+        ETrackingUniverseOrigin::TrackingUniverseStanding => "Standing",
+        ETrackingUniverseOrigin::TrackingUniverseRawAndUncalibrated => "Raw",
+    }
+}
+
+fn get_working_copy(
+    universe: &ETrackingUniverseOrigin,
+    chaperone_mgr: &mut ChaperoneSetupManager,
+) -> Option<Affine3A> {
+    chaperone_mgr.revert_working_copy();
+    let mat = match universe {
+        ETrackingUniverseOrigin::TrackingUniverseStanding => {
+            chaperone_mgr.get_working_standing_zero_pose_to_raw_tracking_pose()
+        }
+        _ => chaperone_mgr.get_working_seated_zero_pose_to_raw_tracking_pose(),
+    };
+    mat.map(|m| m.to_affine())
+}
+
+fn set_working_copy(
+    universe: &ETrackingUniverseOrigin,
+    chaperone_mgr: &mut ChaperoneSetupManager,
+    mat: &Affine3A,
+) {
+    let mat = HmdMatrix34_t::from_affine(mat);
+    match universe {
+        ETrackingUniverseOrigin::TrackingUniverseStanding => {
+            chaperone_mgr.set_working_standing_zero_pose_to_raw_tracking_pose(&mat)
+        }
+        _ => chaperone_mgr.set_working_seated_zero_pose_to_raw_tracking_pose(&mat),
+    };
+}
+
+fn apply_chaperone_offset(offset: Vec3A, chaperone_mgr: &mut ChaperoneSetupManager) {
+    let mut quads = chaperone_mgr.get_live_collision_bounds_info();
+    quads.iter_mut().for_each(|quad| {
+        quad.vCorners.iter_mut().for_each(|corner| {
+            corner.v[0] += offset.x;
+            corner.v[2] += offset.z;
+        });
+    });
+    chaperone_mgr.set_working_collision_bounds_info(quads.as_mut_slice());
+}
+
+fn apply_chaperone_transform(transform: Affine3A, chaperone_mgr: &mut ChaperoneSetupManager) {
+    let mut quads = chaperone_mgr.get_live_collision_bounds_info();
+    quads.iter_mut().for_each(|quad| {
+        quad.vCorners.iter_mut().for_each(|corner| {
+            let coord = transform.transform_point3a(Vec3A::from_slice(&corner.v));
+            corner.v[0] = coord.x;
+            corner.v[2] = coord.z;
+        });
+    });
+    chaperone_mgr.set_working_collision_bounds_info(quads.as_mut_slice());
+}

src/backend/openxr/helpers.rs

@@ -0,0 +1,173 @@
+use anyhow::{bail, ensure};
+use glam::{Affine3A, Quat, Vec3, Vec3A};
+use openxr as xr;
+use xr::OverlaySessionCreateFlagsEXTX;
+
+pub(super) fn init_xr() -> Result<(xr::Instance, xr::SystemId), anyhow::Error> {
+    let entry = xr::Entry::linked();
+
+    let Ok(available_extensions) = entry.enumerate_extensions() else {
+        bail!("Failed to enumerate OpenXR extensions.");
+    };
+    ensure!(
+        available_extensions.khr_vulkan_enable2,
+        "Missing KHR_vulkan_enable2 extension."
+    );
+    ensure!(
+        available_extensions.extx_overlay,
+        "Missing EXTX_overlay extension."
+    );
+
+    let mut enabled_extensions = xr::ExtensionSet::default();
+    enabled_extensions.khr_vulkan_enable2 = true;
+    enabled_extensions.extx_overlay = true;
+
+    //#[cfg(not(debug_assertions))]
+    let layers = [];
+    //#[cfg(debug_assertions)]
+    //let layers = [
+    //    "XR_APILAYER_LUNARG_api_dump",
+    //    "XR_APILAYER_LUNARG_standard_validation",
+    //];
+
+    let Ok(xr_instance) = entry.create_instance(
+        &xr::ApplicationInfo {
+            application_name: "wlx-overlay-s",
+            application_version: 0,
+            engine_name: "wlx-overlay-s",
+            engine_version: 0,
+        },
+        &enabled_extensions,
+        &layers,
+    ) else {
+        bail!("Failed to create OpenXR instance.");
+    };
+
+    let Ok(instance_props) = xr_instance.properties() else {
+        bail!("Failed to query OpenXR instance properties.");
+    };
+    log::info!(
+        "Using OpenXR runtime: {} {}",
+        instance_props.runtime_name,
+        instance_props.runtime_version
+    );
+
+    let Ok(system) = xr_instance.system(xr::FormFactor::HEAD_MOUNTED_DISPLAY) else {
+        bail!("Failed to access OpenXR HMD system.");
+    };
+
+    let vk_target_version_xr = xr::Version::new(1, 1, 0);
+
+    let Ok(reqs) = xr_instance.graphics_requirements::<xr::Vulkan>(system) else {
+        bail!("Failed to query OpenXR Vulkan requirements.");
+    };
+
+    if vk_target_version_xr < reqs.min_api_version_supported
+        || vk_target_version_xr.major() > reqs.max_api_version_supported.major()
+    {
+        bail!(
+            "OpenXR runtime requires Vulkan version > {}, < {}.0.0",
+            reqs.min_api_version_supported,
+            reqs.max_api_version_supported.major() + 1
+        );
+    }
+
+    Ok((xr_instance, system))
+}
+pub(super) unsafe fn create_overlay_session(
+    instance: &xr::Instance,
+    system: xr::SystemId,
+    info: &xr::vulkan::SessionCreateInfo,
+) -> Result<xr::sys::Session, xr::sys::Result> {
+    let overlay = xr::sys::SessionCreateInfoOverlayEXTX {
+        ty: xr::sys::SessionCreateInfoOverlayEXTX::TYPE,
+        next: std::ptr::null(),
+        create_flags: OverlaySessionCreateFlagsEXTX::EMPTY,
+        session_layers_placement: 5,
+    };
+    let binding = xr::sys::GraphicsBindingVulkanKHR {
+        ty: xr::sys::GraphicsBindingVulkanKHR::TYPE,
+        next: &overlay as *const _ as *const _,
+        instance: info.instance,
+        physical_device: info.physical_device,
+        device: info.device,
+        queue_family_index: info.queue_family_index,
+        queue_index: info.queue_index,
+    };
+    let info = xr::sys::SessionCreateInfo {
+        ty: xr::sys::SessionCreateInfo::TYPE,
+        next: &binding as *const _ as *const _,
+        create_flags: Default::default(),
+        system_id: system,
+    };
+    let mut out = xr::sys::Session::NULL;
+    let x = (instance.fp().create_session)(instance.as_raw(), &info, &mut out);
+    if x.into_raw() >= 0 {
+        Ok(out)
+    } else {
+        Err(x)
+    }
+}
+
+pub(super) fn hmd_pose_from_views(views: &[xr::View]) -> Affine3A {
+    let pos = {
+        let pos0: Vec3 = unsafe { std::mem::transmute(views[0].pose.position) };
+        let pos1: Vec3 = unsafe { std::mem::transmute(views[1].pose.position) };
+        (pos0 + pos1) * 0.5
+    };
+    let rot = {
+        let rot0 = unsafe { std::mem::transmute(views[0].pose.orientation) };
+        let rot1 = unsafe { std::mem::transmute(views[1].pose.orientation) };
+        quat_lerp(rot0, rot1, 0.5)
+    };
+
+    Affine3A::from_rotation_translation(rot, pos)
+}
+
+fn quat_lerp(a: Quat, mut b: Quat, t: f32) -> Quat {
+    let l2 = a.dot(b);
+    if l2 < 0.0 {
+        b = -b;
+    }
+
+    Quat::from_xyzw(
+        a.x - t * (a.x - b.x),
+        a.y - t * (a.y - b.y),
+        a.z - t * (a.z - b.z),
+        a.w - t * (a.w - b.w),
+    )
+    .normalize()
+}
+
+pub(super) fn transform_to_norm_quat(transform: &Affine3A) -> Quat {
+    let norm_mat3 = transform
+        .matrix3
+        .mul_scalar(1.0 / transform.matrix3.x_axis.length());
+    Quat::from_mat3a(&norm_mat3).normalize()
+}
+
+pub(super) fn translation_rotation_to_posef(translation: Vec3A, mut rotation: Quat) -> xr::Posef {
+    if !rotation.is_finite() {
+        rotation = Quat::IDENTITY;
+    }
+
+    xr::Posef {
+        orientation: xr::Quaternionf {
+            x: rotation.x,
+            y: rotation.y,
+            z: rotation.z,
+            w: rotation.w,
+        },
+        position: xr::Vector3f {
+            x: translation.x,
+            y: translation.y,
+            z: translation.z,
+        },
+    }
+}
+
+pub(super) fn transform_to_posef(transform: &Affine3A) -> xr::Posef {
+    let translation = transform.translation;
+    let rotation = transform_to_norm_quat(transform);
+    translation_rotation_to_posef(translation, rotation)
+}

src/backend/openxr/input.rs

@@ -0,0 +1,496 @@
+use std::time::{Duration, Instant};
+
+use glam::{bool, Affine3A, Quat, Vec3};
+use openxr as xr;
+
+use crate::{
+    backend::input::{Haptics, Pointer},
+    state::{AppSession, AppState},
+};
+
+use super::XrState;
+
+type XrSession = xr::Session<xr::Vulkan>;
+static DOUBLE_CLICK_TIME: Duration = Duration::from_millis(500);
+
+pub(super) struct DoubleClickCounter {
+    pub(super) last_click: Option<Instant>,
+}
+
+impl DoubleClickCounter {
+    pub(super) fn new() -> Self {
+        Self { last_click: None }
+    }
+
+    // submit a click. returns true if it should count as a double click
+    pub(super) fn click(&mut self) -> bool {
+        let now = Instant::now();
+        let double_click = match self.last_click {
+            Some(last_click) => now - last_click < DOUBLE_CLICK_TIME,
+            None => false,
+        };
+        self.last_click = if double_click { None } else { Some(now) };
+        double_click
+    }
+}
+
+pub(super) struct OpenXrInputSource {
+    action_set: xr::ActionSet,
+    hands: [OpenXrHand; 2],
+}
+
+pub(super) struct OpenXrHand {
+    source: OpenXrHandSource,
+    space: xr::Space,
+}
+
+pub(super) struct OpenXrHandSource {
+    action_pose: xr::Action<xr::Posef>,
+    action_click: xr::Action<f32>,
+    action_grab: xr::Action<f32>,
+    action_scroll: xr::Action<f32>,
+    action_alt_click: xr::Action<f32>,
+    action_show_hide: xr::Action<bool>,
+    action_click_modifier_right: xr::Action<bool>,
+    action_click_modifier_middle: xr::Action<bool>,
+    action_haptics: xr::Action<xr::Haptic>,
+}
+
+impl OpenXrInputSource {
+    pub fn new(xr: &XrState) -> Result<Self, xr::sys::Result> {
+        let mut action_set =
+            xr.session
+                .instance()
+                .create_action_set("wlx-overlay-s", "WlxOverlay-S Actions", 0)?;
+
+        let left_source = OpenXrHandSource::new(&mut action_set, "left")?;
+        let right_source = OpenXrHandSource::new(&mut action_set, "right")?;
+
+        suggest_bindings(&xr.instance, &[&left_source, &right_source])?;
+
+        xr.session.attach_action_sets(&[&action_set])?;
+
+        Ok(Self {
+            action_set,
+            hands: [
+                OpenXrHand::new(xr, left_source)?,
+                OpenXrHand::new(xr, right_source)?,
+            ],
+        })
+    }
+
+    pub fn haptics(&self, xr: &XrState, hand: usize, haptics: &Haptics) {
+        let action = &self.hands[hand].source.action_haptics;
+
+        let duration_nanos = (haptics.duration as f64) * 1_000_000_000.0;
+
+        let _ = action.apply_feedback(
+            &xr.session,
+            xr::Path::NULL,
+            &xr::HapticVibration::new()
+                .amplitude(haptics.intensity)
+                .frequency(haptics.frequency)
+                .duration(xr::Duration::from_nanos(duration_nanos as _)),
+        );
+    }
+
+    pub fn update(&self, xr: &XrState, state: &mut AppState) -> Result<(), xr::sys::Result> {
+        xr.session.sync_actions(&[(&self.action_set).into()])?;
+
+        for i in 0..2 {
+            self.hands[i].update(&mut state.input_state.pointers[i], xr, &state.session)?;
+        }
+        Ok(())
+    }
+}
+
+impl OpenXrHand {
+    pub(super) fn new(xr: &XrState, source: OpenXrHandSource) -> Result<Self, xr::sys::Result> {
+        let space = source.action_pose.create_space(
+            xr.session.clone(),
+            xr::Path::NULL,
+            xr::Posef::IDENTITY,
+        )?;
+
+        Ok(Self { source, space })
+    }
+
+    pub(super) fn update(
+        &self,
+        pointer: &mut Pointer,
+        xr: &XrState,
+        session: &AppSession,
+    ) -> Result<(), xr::sys::Result> {
+        let location = self.space.locate(&xr.stage, xr.predicted_display_time)?;
+        if location
+            .location_flags
+            .contains(xr::SpaceLocationFlags::ORIENTATION_VALID)
+        {
+            let quat = unsafe { std::mem::transmute::<_, Quat>(location.pose.orientation) };
+            let pos = unsafe { std::mem::transmute::<_, Vec3>(location.pose.position) };
+            pointer.pose = Affine3A::from_rotation_translation(quat, pos);
+        }
+
+        let click_sensitivity = if pointer.before.click {
+            session.config.xr_click_sensitivity_release
+        } else {
+            session.config.xr_click_sensitivity
+        };
+
+        pointer.now.click = self
+            .source
+            .action_click
+            .state(&xr.session, xr::Path::NULL)?
+            .current_state
+            > click_sensitivity;
+
+        let grab_sensitivity = if pointer.before.grab {
+            session.config.xr_grab_sensitivity_release
+        } else {
+            session.config.xr_grab_sensitivity
+        };
+
+        pointer.now.grab = self
+            .source
+            .action_grab
+            .state(&xr.session, xr::Path::NULL)?
+            .current_state
+            > grab_sensitivity;
+
+        pointer.now.scroll = self
+            .source
+            .action_scroll
+            .state(&xr.session, xr::Path::NULL)?
+            .current_state;
+
+        let alt_click_sensitivity = if pointer.before.alt_click {
+            session.config.xr_alt_click_sensitivity_release
+        } else {
+            session.config.xr_alt_click_sensitivity
+        };
+
+        pointer.now.alt_click = self
+            .source
+            .action_alt_click
+            .state(&xr.session, xr::Path::NULL)?
+            .current_state
+            > alt_click_sensitivity;
+
+        pointer.now.show_hide = self
+            .source
+            .action_show_hide
+            .state(&xr.session, xr::Path::NULL)?
+            .current_state;
+
+        pointer.now.click_modifier_right = self
+            .source
+            .action_click_modifier_right
+            .state(&xr.session, xr::Path::NULL)?
+            .current_state;
+
+        pointer.now.click_modifier_middle = self
+            .source
+            .action_click_modifier_middle
+            .state(&xr.session, xr::Path::NULL)?
+            .current_state;
+
+        Ok(())
+    }
+}
+
+// supported action types: Haptic, Posef, Vector2f, f32, bool
+impl OpenXrHandSource {
+    pub(super) fn new(action_set: &mut xr::ActionSet, side: &str) -> Result<Self, xr::sys::Result> {
+        let action_pose = action_set.create_action::<xr::Posef>(
+            &format!("{}_hand", side),
+            &format!("{} hand pose", side),
+            &[],
+        )?;
+
+        let action_click = action_set.create_action::<f32>(
+            &format!("{}_click", side),
+            &format!("{} hand click", side),
+            &[],
+        )?;
+        let action_grab = action_set.create_action::<f32>(
+            &format!("{}_grab", side),
+            &format!("{} hand grab", side),
+            &[],
+        )?;
+        let action_scroll = action_set.create_action::<f32>(
+            &format!("{}_scroll", side),
+            &format!("{} hand scroll", side),
+            &[],
+        )?;
+        let action_alt_click = action_set.create_action::<f32>(
+            &format!("{}_alt_click", side),
+            &format!("{} hand alt click", side),
+            &[],
+        )?;
+        let action_show_hide = action_set.create_action::<bool>(
+            &format!("{}_show_hide", side),
+            &format!("{} hand show/hide", side),
+            &[],
+        )?;
+        let action_click_modifier_right = action_set.create_action::<bool>(
+            &format!("{}_click_modifier_right", side),
+            &format!("{} hand right click modifier", side),
+            &[],
+        )?;
+        let action_click_modifier_middle = action_set.create_action::<bool>(
+            &format!("{}_click_modifier_middle", side),
+            &format!("{} hand middle click modifier", side),
+            &[],
+        )?;
+        let action_haptics = action_set.create_action::<xr::Haptic>(
+            &format!("{}_haptics", side),
+            &format!("{} hand haptics", side),
+            &[],
+        )?;
+
+        Ok(Self {
+            action_pose,
+            action_click,
+            action_grab,
+            action_scroll,
+            action_alt_click,
+            action_show_hide,
+            action_click_modifier_right,
+            action_click_modifier_middle,
+            action_haptics,
+        })
+    }
+}
+
+fn suggest_bindings(
+    instance: &xr::Instance,
+    hands: &[&OpenXrHandSource; 2],
+) -> Result<(), xr::sys::Result> {
+    let path = instance.string_to_path("/interaction_profiles/khr/simple_controller")?;
+
+    // not fully functional, but helpful for debugging
+    instance.suggest_interaction_profile_bindings(
+        path,
+        &[
+            xr::Binding::new(
+                &hands[0].action_pose,
+                instance.string_to_path("/user/hand/left/input/aim/pose")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_pose,
+                instance.string_to_path("/user/hand/right/input/aim/pose")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_click,
+                instance.string_to_path("/user/hand/left/input/select/click")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_click,
+                instance.string_to_path("/user/hand/right/input/select/click")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_show_hide,
+                instance.string_to_path("/user/hand/left/input/menu/click")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_haptics,
+                instance.string_to_path("/user/hand/left/output/haptic")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_haptics,
+                instance.string_to_path("/user/hand/right/output/haptic")?,
+            ),
+        ],
+    )?;
+
+    let path = instance.string_to_path("/interaction_profiles/oculus/touch_controller")?;
+    instance.suggest_interaction_profile_bindings(
+        path,
+        &[
+            xr::Binding::new(
+                &hands[0].action_pose,
+                instance.string_to_path("/user/hand/left/input/aim/pose")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_pose,
+                instance.string_to_path("/user/hand/right/input/aim/pose")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_click,
+                instance.string_to_path("/user/hand/left/input/trigger/value")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_click,
+                instance.string_to_path("/user/hand/right/input/trigger/value")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_grab,
+                instance.string_to_path("/user/hand/left/input/squeeze/value")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_grab,
+                instance.string_to_path("/user/hand/right/input/squeeze/value")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_scroll,
+                instance.string_to_path("/user/hand/left/input/thumbstick/y")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_scroll,
+                instance.string_to_path("/user/hand/right/input/thumbstick/y")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_show_hide,
+                instance.string_to_path("/user/hand/left/input/y/click")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_click_modifier_right,
+                instance.string_to_path("/user/hand/left/input/y/touch")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_click_modifier_right,
+                instance.string_to_path("/user/hand/right/input/b/touch")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_click_modifier_middle,
+                instance.string_to_path("/user/hand/left/input/x/touch")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_click_modifier_middle,
+                instance.string_to_path("/user/hand/right/input/a/touch")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_haptics,
+                instance.string_to_path("/user/hand/left/output/haptic")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_haptics,
+                instance.string_to_path("/user/hand/right/output/haptic")?,
+            ),
+        ],
+    )?;
+
+    let path = instance.string_to_path("/interaction_profiles/valve/index_controller")?;
+    instance.suggest_interaction_profile_bindings(
+        path,
+        &[
+            xr::Binding::new(
+                &hands[0].action_pose,
+                instance.string_to_path("/user/hand/left/input/aim/pose")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_pose,
+                instance.string_to_path("/user/hand/right/input/aim/pose")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_click,
+                instance.string_to_path("/user/hand/left/input/trigger/value")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_click,
+                instance.string_to_path("/user/hand/right/input/trigger/value")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_grab,
+                instance.string_to_path("/user/hand/left/input/squeeze/force")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_grab,
+                instance.string_to_path("/user/hand/right/input/squeeze/force")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_scroll,
+                instance.string_to_path("/user/hand/left/input/thumbstick/y")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_scroll,
+                instance.string_to_path("/user/hand/right/input/thumbstick/y")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_alt_click,
+                instance.string_to_path("/user/hand/left/input/trackpad/force")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_alt_click,
+                instance.string_to_path("/user/hand/right/input/trackpad/force")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_show_hide,
+                instance.string_to_path("/user/hand/left/input/b/click")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_click_modifier_right,
+                instance.string_to_path("/user/hand/left/input/b/touch")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_click_modifier_right,
+                instance.string_to_path("/user/hand/right/input/b/touch")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_click_modifier_middle,
+                instance.string_to_path("/user/hand/left/input/a/touch")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_click_modifier_middle,
+                instance.string_to_path("/user/hand/right/input/a/touch")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_haptics,
+                instance.string_to_path("/user/hand/left/output/haptic")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_haptics,
+                instance.string_to_path("/user/hand/right/output/haptic")?,
+            ),
+        ],
+    )?;
+
+    let path = instance.string_to_path("/interaction_profiles/htc/vive_controller")?;
+    instance.suggest_interaction_profile_bindings(
+        path,
+        &[
+            xr::Binding::new(
+                &hands[0].action_pose,
+                instance.string_to_path("/user/hand/left/input/aim/pose")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_pose,
+                instance.string_to_path("/user/hand/right/input/aim/pose")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_click,
+                instance.string_to_path("/user/hand/right/input/trigger/value")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_grab,
+                instance.string_to_path("/user/hand/left/input/squeeze/click")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_grab,
+                instance.string_to_path("/user/hand/right/input/squeeze/click")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_scroll,
+                instance.string_to_path("/user/hand/left/input/trackpad/y")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_scroll,
+                instance.string_to_path("/user/hand/right/input/trackpad/y")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_show_hide,
+                instance.string_to_path("/user/hand/left/input/menu/click")?,
+            ),
+            xr::Binding::new(
+                &hands[0].action_haptics,
+                instance.string_to_path("/user/hand/left/output/haptic")?,
+            ),
+            xr::Binding::new(
+                &hands[1].action_haptics,
+                instance.string_to_path("/user/hand/right/output/haptic")?,
+            ),
+        ],
+    )?;
+
+    Ok(())
+}

src/backend/openxr/lines.rs

@@ -0,0 +1,183 @@
+use glam::{Affine3A, Vec3, Vec3A};
+use idmap::IdMap;
+use openxr as xr;
+use std::{
+    f32::consts::PI,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
+    usize,
+};
+
+use vulkano::{command_buffer::CommandBufferUsage, format::Format, image::view::ImageView};
+
+use crate::{
+    backend::openxr::helpers,
+    graphics::{WlxCommandBuffer, WlxGraphics},
+};
+
+use super::{
+    swapchain::{create_swapchain_render_data, SwapchainRenderData},
+    CompositionLayer, XrState,
+};
+
+static AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(1);
+pub(super) const LINE_WIDTH: f32 = 0.002;
+
+pub(super) struct LinePool {
+    lines: IdMap<usize, LineContainer>,
+    colors: Vec<Arc<ImageView>>,
+}
+
+impl LinePool {
+    pub(super) fn new(graphics: Arc<WlxGraphics>) -> anyhow::Result<Self> {
+        let mut command_buffer =
+            graphics.create_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+
+        // TODO customizable colors
+        let colors = [
+            [0xff, 0xff, 0xff, 0xff],
+            [0x00, 0x60, 0x80, 0xff],
+            [0xB0, 0x30, 0x00, 0xff],
+            [0x60, 0x00, 0x80, 0xff],
+            [0xff, 0x00, 0x00, 0xff],
+        ];
+
+        let views: anyhow::Result<Vec<Arc<ImageView>>> = colors
+            .into_iter()
+            .map(
+                |color| match command_buffer.texture2d(1, 1, Format::R8G8B8A8_UNORM, &color) {
+                    Ok(tex) => ImageView::new_default(tex).map_err(|e| anyhow::anyhow!(e)),
+                    Err(e) => Err(e),
+                },
+            )
+            .collect();
+
+        command_buffer.build_and_execute_now()?;
+
+        Ok(LinePool {
+            lines: IdMap::new(),
+            colors: views?,
+        })
+    }
+
+    pub(super) fn allocate(
+        &mut self,
+        xr: &XrState,
+        graphics: Arc<WlxGraphics>,
+    ) -> anyhow::Result<usize> {
+        let id = AUTO_INCREMENT.fetch_add(1, Ordering::Relaxed);
+
+        let srd = create_swapchain_render_data(xr, graphics, [1, 1, 1])?;
+        self.lines.insert(
+            id,
+            LineContainer {
+                swapchain: srd,
+                maybe_line: None,
+            },
+        );
+        Ok(id)
+    }
+
+    pub(super) fn draw_from(
+        &mut self,
+        id: usize,
+        mut from: Affine3A,
+        len: f32,
+        color: usize,
+        hmd: &Affine3A,
+    ) {
+        if len < 0.01 {
+            return;
+        }
+
+        debug_assert!(color < self.colors.len());
+
+        let Some(line) = self.lines.get_mut(id) else {
+            log::warn!("Line {} not found", id);
+            return;
+        };
+
+        let rotation = Affine3A::from_axis_angle(Vec3::X, PI * 1.5);
+
+        from.translation += from.transform_vector3a(Vec3A::NEG_Z) * (len * 0.5);
+        let mut transform = from * rotation;
+
+        let to_hmd = hmd.translation - from.translation;
+        let sides = [Vec3A::Z, Vec3A::X, Vec3A::NEG_Z, Vec3A::NEG_X];
+        let rotations = [
+            Affine3A::IDENTITY,
+            Affine3A::from_axis_angle(Vec3::Y, PI * 0.5),
+            Affine3A::from_axis_angle(Vec3::Y, PI * -1.0),
+            Affine3A::from_axis_angle(Vec3::Y, PI * 1.5),
+        ];
+        let mut closest = (0, 0.0);
+        for (i, &side) in sides.iter().enumerate() {
+            let dot = to_hmd.dot(transform.transform_vector3a(side));
+            if i == 0 || dot > closest.1 {
+                closest = (i, dot);
+            }
+        }
+
+        transform *= rotations[closest.0];
+
+        let posef = helpers::transform_to_posef(&transform);
+
+        line.maybe_line = Some(Line {
+            view: self.colors[color].clone(),
+            pose: posef,
+            length: len,
+        });
+    }
+
+    pub(super) fn present_xr<'a>(
+        &'a mut self,
+        xr: &'a XrState,
+        command_buffer: &mut WlxCommandBuffer,
+    ) -> anyhow::Result<Vec<CompositionLayer>> {
+        let mut quads = Vec::new();
+
+        for line in self.lines.values_mut() {
+            if let Some(inner) = line.maybe_line.take() {
+                let quad = xr::CompositionLayerQuad::new()
+                    .pose(inner.pose)
+                    .sub_image(line.swapchain.acquire_present_release(
+                        command_buffer,
+                        inner.view,
+                        1.0,
+                    )?)
+                    .eye_visibility(xr::EyeVisibility::BOTH)
+                    .layer_flags(xr::CompositionLayerFlags::CORRECT_CHROMATIC_ABERRATION)
+                    .space(&xr.stage)
+                    .size(xr::Extent2Df {
+                        width: LINE_WIDTH,
+                        height: inner.length,
+                    });
+
+                quads.push(CompositionLayer::Quad(quad));
+            }
+        }
+
+        Ok(quads)
+    }
+
+    /// the number of lines that are waiting to be drawn
+    pub(super) fn num_pending(&self) -> usize {
+        self.lines
+            .values()
+            .filter(|l| l.maybe_line.is_some())
+            .count()
+    }
+}
+
+pub(super) struct Line {
+    pub(super) view: Arc<ImageView>,
+    pub(super) pose: xr::Posef,
+    pub(super) length: f32,
+}
+
+struct LineContainer {
+    swapchain: SwapchainRenderData,
+    maybe_line: Option<Line>,
+}

src/backend/openxr/mod.rs

@@ -0,0 +1,368 @@
+use std::{
+    collections::VecDeque,
+    sync::{
+        atomic::{AtomicBool, AtomicUsize, Ordering},
+        Arc,
+    },
+    time::Duration,
+};
+
+use glam::{Affine3A, Vec3};
+use openxr as xr;
+use vulkano::{command_buffer::CommandBufferUsage, Handle, VulkanObject};
+
+use crate::{
+    backend::{
+        common::{OverlayContainer, TaskType},
+        input::interact,
+        notifications::NotificationManager,
+        openxr::{input::DoubleClickCounter, lines::LinePool, overlay::OpenXrOverlayData},
+        overlay::OverlayData,
+    },
+    graphics::WlxGraphics,
+    overlays::watch::{watch_fade, WATCH_NAME},
+    state::AppState,
+};
+
+use super::common::BackendError;
+
+mod helpers;
+mod input;
+mod lines;
+mod overlay;
+mod swapchain;
+
+const VIEW_TYPE: xr::ViewConfigurationType = xr::ViewConfigurationType::PRIMARY_STEREO;
+const VIEW_COUNT: u32 = 2;
+static FRAME_COUNTER: AtomicUsize = AtomicUsize::new(0);
+
+struct XrState {
+    instance: xr::Instance,
+    system: xr::SystemId,
+    session: xr::Session<xr::Vulkan>,
+    predicted_display_time: xr::Time,
+    stage: Arc<xr::Space>,
+}
+
+pub fn openxr_run(running: Arc<AtomicBool>) -> Result<(), BackendError> {
+    let (xr_instance, system) = match helpers::init_xr() {
+        Ok((xr_instance, system)) => (xr_instance, system),
+        Err(e) => {
+            log::warn!("Will not use OpenXR: {}", e);
+            return Err(BackendError::NotSupported);
+        }
+    };
+
+    let environment_blend_mode =
+        xr_instance.enumerate_environment_blend_modes(system, VIEW_TYPE)?[0];
+    log::info!("Using environment blend mode: {:?}", environment_blend_mode);
+
+    let mut app_state = {
+        let graphics = WlxGraphics::new_openxr(xr_instance.clone(), system)?;
+        AppState::from_graphics(graphics)?
+    };
+
+    let mut overlays = OverlayContainer::<OpenXrOverlayData>::new(&mut app_state)?;
+    let mut lines = LinePool::new(app_state.graphics.clone())?;
+
+    let mut notifications = NotificationManager::new();
+    notifications.run_dbus();
+    notifications.run_udp();
+
+    let mut delete_queue = vec![];
+
+    #[cfg(feature = "osc")]
+    let mut osc_sender =
+        crate::backend::osc::OscSender::new(app_state.session.config.osc_out_port).ok();
+
+    app_state.hid_provider.set_desktop_extent(overlays.extent);
+
+    let (session, mut frame_wait, mut frame_stream) = unsafe {
+        let raw_session = helpers::create_overlay_session(
+            &xr_instance,
+            system,
+            &xr::vulkan::SessionCreateInfo {
+                instance: app_state.graphics.instance.handle().as_raw() as _,
+                physical_device: app_state
+                    .graphics
+                    .device
+                    .physical_device()
+                    .handle()
+                    .as_raw() as _,
+                device: app_state.graphics.device.handle().as_raw() as _,
+                queue_family_index: app_state.graphics.queue.queue_family_index(),
+                queue_index: 0,
+            },
+        )?;
+        xr::Session::from_raw(xr_instance.clone(), raw_session, Box::new(()))
+    };
+
+    let stage =
+        session.create_reference_space(xr::ReferenceSpaceType::STAGE, xr::Posef::IDENTITY)?;
+
+    let mut xr_state = XrState {
+        instance: xr_instance,
+        system,
+        session,
+        predicted_display_time: xr::Time::from_nanos(0),
+        stage: Arc::new(stage),
+    };
+
+    let pointer_lines = [
+        lines.allocate(&xr_state, app_state.graphics.clone())?,
+        lines.allocate(&xr_state, app_state.graphics.clone())?,
+    ];
+
+    let watch_id = overlays.get_by_name(WATCH_NAME).unwrap().state.id; // want panic
+
+    let input_source = input::OpenXrInputSource::new(&xr_state)?;
+
+    let mut session_running = false;
+    let mut event_storage = xr::EventDataBuffer::new();
+
+    let mut show_hide_counter = DoubleClickCounter::new();
+    let mut due_tasks = VecDeque::with_capacity(4);
+
+    'main_loop: loop {
+        let cur_frame = FRAME_COUNTER.fetch_add(1, Ordering::Relaxed);
+
+        if !running.load(Ordering::Relaxed) {
+            log::warn!("Received shutdown signal.");
+            match xr_state.session.request_exit() {
+                Ok(_) => log::info!("OpenXR session exit requested."),
+                Err(xr::sys::Result::ERROR_SESSION_NOT_RUNNING) => break 'main_loop,
+                Err(e) => {
+                    log::error!("Failed to request OpenXR session exit: {}", e);
+                    break 'main_loop;
+                }
+            }
+        }
+
+        while let Some(event) = xr_state.instance.poll_event(&mut event_storage)? {
+            use xr::Event::*;
+            match event {
+                SessionStateChanged(e) => {
+                    // Session state change is where we can begin and end sessions, as well as
+                    // find quit messages!
+                    log::info!("entered state {:?}", e.state());
+                    match e.state() {
+                        xr::SessionState::READY => {
+                            xr_state.session.begin(VIEW_TYPE)?;
+                            session_running = true;
+                        }
+                        xr::SessionState::STOPPING => {
+                            xr_state.session.end()?;
+                            session_running = false;
+                        }
+                        xr::SessionState::EXITING | xr::SessionState::LOSS_PENDING => {
+                            break 'main_loop;
+                        }
+                        _ => {}
+                    }
+                }
+                InstanceLossPending(_) => {
+                    break 'main_loop;
+                }
+                EventsLost(e) => {
+                    log::warn!("lost {} events", e.lost_event_count());
+                }
+                _ => {}
+            }
+        }
+
+        if !session_running {
+            std::thread::sleep(Duration::from_millis(100));
+            continue 'main_loop;
+        }
+
+        let xr_frame_state = frame_wait.wait()?;
+        frame_stream.begin()?;
+
+        xr_state.predicted_display_time = xr_frame_state.predicted_display_time;
+
+        if !xr_frame_state.should_render {
+            frame_stream.end(
+                xr_frame_state.predicted_display_time,
+                environment_blend_mode,
+                &[],
+            )?;
+            continue 'main_loop;
+        }
+
+        app_state.input_state.pre_update();
+        input_source.update(&xr_state, &mut app_state)?;
+        app_state.input_state.post_update();
+
+        if app_state
+            .input_state
+            .pointers
+            .iter()
+            .any(|p| p.now.show_hide && !p.before.show_hide)
+            && show_hide_counter.click()
+        {
+            overlays.show_hide(&mut app_state);
+        }
+
+        watch_fade(&mut app_state, overlays.mut_by_id(watch_id).unwrap()); // want panic
+
+        for o in overlays.iter_mut() {
+            o.after_input(&mut app_state)?;
+        }
+
+        #[cfg(feature = "osc")]
+        if let Some(ref mut sender) = osc_sender {
+            let _ = sender.send_params(&overlays);
+        };
+
+        let (_, views) = xr_state.session.locate_views(
+            VIEW_TYPE,
+            xr_frame_state.predicted_display_time,
+            &xr_state.stage,
+        )?;
+
+        app_state.input_state.hmd = helpers::hmd_pose_from_views(&views);
+
+        overlays
+            .iter_mut()
+            .for_each(|o| o.state.auto_movement(&mut app_state));
+
+        let lengths_haptics = interact(&mut overlays, &mut app_state);
+        for (idx, (len, haptics)) in lengths_haptics.iter().enumerate() {
+            lines.draw_from(
+                pointer_lines[idx],
+                app_state.input_state.pointers[idx].pose,
+                *len,
+                app_state.input_state.pointers[idx].interaction.mode as usize + 1,
+                &app_state.input_state.hmd,
+            );
+            if let Some(haptics) = haptics {
+                input_source.haptics(&xr_state, idx, haptics);
+            }
+        }
+
+        let watch = overlays.mut_by_id(watch_id).unwrap(); // want panic
+        let watch_transform = watch.state.transform;
+        if !watch.state.want_visible {
+            watch.state.want_visible = true;
+            watch.state.transform = Affine3A::from_scale(Vec3 {
+                x: 0.001,
+                y: 0.001,
+                z: 0.001,
+            });
+        }
+
+        let mut layers = vec![];
+        let mut command_buffer = app_state
+            .graphics
+            .create_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+
+        for o in overlays.iter_mut() {
+            if !o.state.want_visible {
+                continue;
+            }
+
+            if !o.data.init {
+                o.init(&mut app_state)?;
+                o.data.init = true;
+            }
+
+            o.render(&mut app_state)?;
+
+            let dist_sq = (app_state.input_state.hmd.translation - o.state.transform.translation)
+                .length_squared();
+
+            if !dist_sq.is_normal() {
+                continue;
+            }
+
+            let maybe_layer = o.present_xr(&xr_state, &mut command_buffer)?;
+            if let CompositionLayer::None = maybe_layer {
+                continue;
+            }
+            layers.push((dist_sq, maybe_layer));
+        }
+
+        for maybe_layer in lines.present_xr(&xr_state, &mut command_buffer)? {
+            if let CompositionLayer::None = maybe_layer {
+                continue;
+            }
+            layers.push((0.0, maybe_layer));
+        }
+
+        command_buffer.build_and_execute_now()?;
+
+        layers.sort_by(|a, b| b.0.total_cmp(&a.0));
+
+        let frame_ref = layers
+            .iter()
+            .map(|f| match f.1 {
+                CompositionLayer::Quad(ref l) => l as &xr::CompositionLayerBase<xr::Vulkan>,
+                CompositionLayer::Cylinder(ref l) => l as &xr::CompositionLayerBase<xr::Vulkan>,
+                CompositionLayer::None => unreachable!(),
+            })
+            .collect::<Vec<_>>();
+
+        frame_stream.end(
+            xr_state.predicted_display_time,
+            environment_blend_mode,
+            &frame_ref,
+        )?;
+
+        notifications.submit_pending(&mut app_state);
+
+        app_state.tasks.retrieve_due(&mut due_tasks);
+        while let Some(task) = due_tasks.pop_front() {
+            match task {
+                TaskType::Global(f) => f(&mut app_state),
+                TaskType::Overlay(sel, f) => {
+                    if let Some(o) = overlays.mut_by_selector(&sel) {
+                        f(&mut app_state, &mut o.state);
+                    }
+                }
+                TaskType::CreateOverlay(sel, f) => {
+                    let None = overlays.mut_by_selector(&sel) else {
+                        continue;
+                    };
+
+                    let Some((mut state, backend)) = f(&mut app_state) else {
+                        continue;
+                    };
+                    state.birthframe = cur_frame;
+
+                    overlays.add(OverlayData {
+                        state,
+                        backend,
+                        ..Default::default()
+                    });
+                }
+                TaskType::DropOverlay(sel) => {
+                    if let Some(o) = overlays.mut_by_selector(&sel) {
+                        if o.state.birthframe < cur_frame {
+                            let o = overlays.remove_by_selector(&sel);
+                            // set for deletion after all images are done showing
+                            delete_queue.push((o, cur_frame + 5));
+                        }
+                    }
+                }
+                TaskType::System(_task) => {
+                    // Not implemented
+                }
+            }
+        }
+
+        delete_queue.retain(|(_, frame)| *frame > cur_frame);
+
+        app_state.hid_provider.on_new_frame();
+
+        let watch = overlays.mut_by_id(watch_id).unwrap(); // want panic
+        watch.state.transform = watch_transform;
+    }
+
+    Ok(())
+}
+
+pub(super) enum CompositionLayer<'a> {
+    None,
+    Quad(xr::CompositionLayerQuad<'a, xr::Vulkan>),
+    Cylinder(xr::CompositionLayerCylinderKHR<'a, xr::Vulkan>),
+}

src/backend/openxr/overlay.rs

@@ -0,0 +1,114 @@
+use glam::Vec3A;
+use openxr as xr;
+use std::{f32::consts::PI, sync::Arc};
+use xr::{CompositionLayerFlags, EyeVisibility};
+
+use super::{helpers, swapchain::SwapchainRenderData, CompositionLayer, XrState};
+use crate::{
+    backend::{openxr::swapchain::create_swapchain_render_data, overlay::OverlayData},
+    graphics::WlxCommandBuffer,
+    state::AppState,
+};
+use vulkano::image::view::ImageView;
+
+#[derive(Default)]
+pub struct OpenXrOverlayData {
+    last_view: Option<Arc<ImageView>>,
+    last_visible: bool,
+    pub(super) swapchain: Option<SwapchainRenderData>,
+    pub(super) init: bool,
+}
+
+impl OverlayData<OpenXrOverlayData> {
+    pub(super) fn present_xr<'a>(
+        &'a mut self,
+        xr: &'a XrState,
+        command_buffer: &mut WlxCommandBuffer,
+    ) -> anyhow::Result<CompositionLayer> {
+        if let Some(new_view) = self.view() {
+            self.data.last_view = Some(new_view);
+        }
+
+        let my_view = if let Some(view) = self.data.last_view.as_ref() {
+            view.clone()
+        } else {
+            log::warn!("{}: Will not show - image not ready", self.state.name);
+            return Ok(CompositionLayer::None);
+        };
+        let extent = my_view.image().extent();
+
+        let data = match self.data.swapchain {
+            Some(ref mut data) => data,
+            None => {
+                let srd =
+                    create_swapchain_render_data(xr, command_buffer.graphics.clone(), extent)?;
+                log::debug!(
+                    "{}: Created swapchain {}x{}, {} images, {} MB",
+                    self.state.name,
+                    extent[0],
+                    extent[1],
+                    srd.images.len(),
+                    extent[0] * extent[1] * 4 * srd.images.len() as u32 / 1024 / 1024
+                );
+                self.data.swapchain = Some(srd);
+                self.data.swapchain.as_mut().unwrap() //safe
+            }
+        };
+
+        let sub_image = data.acquire_present_release(command_buffer, my_view, self.state.alpha)?;
+
+        let aspect_ratio = extent[1] as f32 / extent[0] as f32;
+        let (scale_x, scale_y) = if aspect_ratio < 1.0 {
+            let major = self.state.transform.matrix3.col(0).length();
+            (major, major * aspect_ratio)
+        } else {
+            let major = self.state.transform.matrix3.col(1).length();
+            (major / aspect_ratio, major)
+        };
+
+        if let Some(curvature) = self.state.curvature {
+            let radius = scale_x / (2.0 * PI * curvature);
+            let quat = helpers::transform_to_norm_quat(&self.state.transform);
+            let center_point = self.state.transform.translation + quat.mul_vec3a(Vec3A::Z * radius);
+
+            let posef = helpers::translation_rotation_to_posef(center_point, quat);
+            let angle = 2.0 * (scale_x / (2.0 * radius));
+
+            let cylinder = xr::CompositionLayerCylinderKHR::new()
+                .pose(posef)
+                .sub_image(sub_image)
+                .eye_visibility(EyeVisibility::BOTH)
+                .layer_flags(CompositionLayerFlags::CORRECT_CHROMATIC_ABERRATION)
+                .space(&xr.stage)
+                .radius(radius)
+                .central_angle(angle)
+                .aspect_ratio(aspect_ratio);
+            Ok(CompositionLayer::Cylinder(cylinder))
+        } else {
+            let posef = helpers::transform_to_posef(&self.state.transform);
+            let quad = xr::CompositionLayerQuad::new()
+                .pose(posef)
+                .sub_image(sub_image)
+                .eye_visibility(EyeVisibility::BOTH)
+                .layer_flags(CompositionLayerFlags::CORRECT_CHROMATIC_ABERRATION)
+                .space(&xr.stage)
+                .size(xr::Extent2Df {
+                    width: scale_x,
+                    height: scale_y,
+                });
+            Ok(CompositionLayer::Quad(quad))
+        }
+    }
+
+    pub(super) fn after_input(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        if self.data.last_visible != self.state.want_visible {
+            if self.state.want_visible {
+                self.backend.resume(app)?;
+            } else {
+                self.backend.pause(app)?;
+            }
+        }
+        self.data.last_visible = self.state.want_visible;
+        Ok(())
+    }
+}

src/backend/openxr/swapchain.rs

@@ -0,0 +1,127 @@
+use std::sync::Arc;
+
+use anyhow::bail;
+use ash::vk;
+use openxr as xr;
+
+use smallvec::SmallVec;
+use vulkano::{
+    image::{sampler::Filter, sys::RawImage, view::ImageView, ImageCreateInfo, ImageUsage},
+    pipeline::graphics::color_blend::AttachmentBlend,
+    Handle,
+};
+
+use crate::graphics::{WlxCommandBuffer, WlxGraphics, WlxPipeline, WlxPipelineDynamic};
+
+use super::XrState;
+
+pub(super) fn create_swapchain_render_data(
+    xr: &XrState,
+    graphics: Arc<WlxGraphics>,
+    extent: [u32; 3],
+) -> anyhow::Result<SwapchainRenderData> {
+    let swapchain = xr.session.create_swapchain(&xr::SwapchainCreateInfo {
+        create_flags: xr::SwapchainCreateFlags::EMPTY,
+        usage_flags: xr::SwapchainUsageFlags::COLOR_ATTACHMENT | xr::SwapchainUsageFlags::SAMPLED,
+        format: graphics.native_format as _,
+        sample_count: 1,
+        width: extent[0],
+        height: extent[1],
+        face_count: 1,
+        array_size: 1,
+        mip_count: 1,
+    })?;
+
+    let Ok(shaders) = graphics.shared_shaders.read() else {
+        bail!("Failed to lock shared shaders for reading");
+    };
+    let pipeline = graphics.create_pipeline_dynamic(
+        shaders.get("vert_common").unwrap().clone(), // want panic
+        shaders.get("frag_srgb").unwrap().clone(),   // want panic
+        graphics.native_format,
+        Some(AttachmentBlend::alpha()),
+    )?;
+
+    let images = swapchain
+        .enumerate_images()?
+        .into_iter()
+        .map(|handle| {
+            let vk_image = vk::Image::from_raw(handle);
+            // thanks @yshui
+            let raw_image = unsafe {
+                RawImage::from_handle_borrowed(
+                    graphics.device.clone(),
+                    vk_image,
+                    ImageCreateInfo {
+                        format: graphics.native_format,
+                        extent,
+                        usage: ImageUsage::COLOR_ATTACHMENT,
+                        ..Default::default()
+                    },
+                )?
+            };
+            // SAFETY: OpenXR guarantees that the image is a swapchain image, thus has memory backing it.
+            let image = Arc::new(unsafe { raw_image.assume_bound() });
+            Ok(ImageView::new_default(image)?)
+        })
+        .collect::<anyhow::Result<SmallVec<[Arc<ImageView>; 4]>>>()?;
+
+    Ok(SwapchainRenderData {
+        swapchain,
+        pipeline,
+        images,
+        extent,
+    })
+}
+
+pub(super) struct SwapchainRenderData {
+    pub(super) swapchain: xr::Swapchain<xr::Vulkan>,
+    pub(super) pipeline: Arc<WlxPipeline<WlxPipelineDynamic>>,
+    pub(super) extent: [u32; 3],
+    pub(super) images: SmallVec<[Arc<ImageView>; 4]>,
+}
+
+impl SwapchainRenderData {
+    pub(super) fn acquire_present_release(
+        &mut self,
+        command_buffer: &mut WlxCommandBuffer,
+        view: Arc<ImageView>,
+        alpha: f32,
+    ) -> anyhow::Result<xr::SwapchainSubImage<xr::Vulkan>> {
+        let idx = self.swapchain.acquire_image()? as usize;
+        self.swapchain.wait_image(xr::Duration::INFINITE)?;
+
+        let render_target = &mut self.images[idx];
+        command_buffer.begin_rendering(render_target.clone())?;
+
+        let target_extent = render_target.image().extent();
+
+        let set0 = self
+            .pipeline
+            .uniform_sampler(0, view.clone(), Filter::Linear)?;
+
+        let set1 = self.pipeline.uniform_buffer(1, vec![alpha])?;
+
+        let pass = self.pipeline.create_pass(
+            [target_extent[0] as _, target_extent[1] as _],
+            command_buffer.graphics.quad_verts.clone(),
+            command_buffer.graphics.quad_indices.clone(),
+            vec![set0, set1],
+        )?;
+        command_buffer.run_ref(&pass)?;
+        command_buffer.end_rendering()?;
+
+        self.swapchain.release_image()?;
+
+        Ok(xr::SwapchainSubImage::new()
+            .swapchain(&self.swapchain)
+            .image_rect(xr::Rect2Di {
+                offset: xr::Offset2Di { x: 0, y: 0 },
+                extent: xr::Extent2Di {
+                    width: target_extent[0] as _,
+                    height: target_extent[1] as _,
+                },
+            })
+            .image_array_index(0))
+    }
+}

src/backend/osc.rs

@@ -0,0 +1,95 @@
+use std::{
+    net::{IpAddr, Ipv4Addr, SocketAddr, UdpSocket},
+    time::Instant,
+};
+
+use anyhow::bail;
+use rosc::{OscMessage, OscPacket, OscType};
+
+use crate::overlays::{keyboard::KEYBOARD_NAME, watch::WATCH_NAME};
+
+use super::common::OverlayContainer;
+
+pub struct OscSender {
+    last_sent: Instant,
+    upstream: UdpSocket,
+}
+
+impl OscSender {
+    pub fn new(send_port: u16) -> anyhow::Result<Self> {
+        let ip = IpAddr::V4(Ipv4Addr::LOCALHOST);
+
+        let Ok(upstream) = UdpSocket::bind("0.0.0.0:0") else {
+            bail!("Failed to bind UDP socket - OSC will not function.");
+        };
+
+        let Ok(_) = upstream.connect(SocketAddr::new(ip, send_port)) else {
+            bail!("Failed to connect UDP socket - OSC will not function.");
+        };
+
+        Ok(Self {
+            upstream,
+            last_sent: Instant::now(),
+        })
+    }
+
+    pub fn send_message(&self, addr: String, args: Vec<OscType>) -> anyhow::Result<()> {
+        let packet = OscPacket::Message(OscMessage { addr, args });
+        let Ok(bytes) = rosc::encoder::encode(&packet) else {
+            bail!("Could not encode OSC packet.");
+        };
+
+        let Ok(_) = self.upstream.send(&bytes) else {
+            bail!("Could not send OSC packet.");
+        };
+
+        Ok(())
+    }
+
+    pub fn send_params<D>(&mut self, overlays: &OverlayContainer<D>) -> anyhow::Result<()>
+    where
+        D: Default,
+    {
+        if self.last_sent.elapsed().as_millis() < 100 {
+            return Ok(());
+        }
+
+        let mut num_overlays = 0;
+        let mut has_keyboard = false;
+        let mut has_wrist = false;
+
+        for o in overlays.iter() {
+            if !o.state.want_visible {
+                continue;
+            }
+            match o.state.name.as_ref() {
+                WATCH_NAME => has_wrist = true,
+                KEYBOARD_NAME => has_keyboard = true,
+                _ => {
+                    if o.state.interactable {
+                        num_overlays += 1
+                    }
+                }
+            }
+        }
+
+        self.send_message(
+            "/avatar/parameters/isOverlayOpen".into(),
+            vec![OscType::Bool(num_overlays > 0)],
+        )?;
+        self.send_message(
+            "/avatar/parameters/isKeyboardOpen".into(),
+            vec![OscType::Bool(has_keyboard)],
+        )?;
+        self.send_message(
+            "/avatar/parameters/isWristVisible".into(),
+            vec![OscType::Bool(has_wrist)],
+        )?;
+        self.send_message(
+            "/avatar/parameters/openOverlayCount".into(),
+            vec![OscType::Int(num_overlays)],
+        )?;
+
+        Ok(())
+    }
+}

src/backend/overlay.rs

@@ -0,0 +1,307 @@
+use std::{
+    f32::consts::PI,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
+};
+
+use anyhow::Ok;
+use glam::{Affine2, Affine3A, Mat3A, Quat, Vec2, Vec3, Vec3A};
+use vulkano::image::view::ImageView;
+
+use crate::state::AppState;
+
+use super::input::{DummyInteractionHandler, Haptics, InteractionHandler, PointerHit};
+
+static AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(0);
+
+pub trait OverlayBackend: OverlayRenderer + InteractionHandler {}
+
+pub struct OverlayState {
+    pub id: usize,
+    pub name: Arc<str>,
+    pub want_visible: bool,
+    pub show_hide: bool,
+    pub grabbable: bool,
+    pub interactable: bool,
+    pub recenter: bool,
+    pub dirty: bool,
+    pub alpha: f32,
+    pub transform: Affine3A,
+    pub saved_point: Option<Vec3A>,
+    pub saved_scale: Option<f32>,
+    pub spawn_scale: f32, // aka width
+    pub spawn_point: Vec3A,
+    pub spawn_rotation: Quat,
+    pub relative_to: RelativeTo,
+    pub curvature: Option<f32>,
+    pub primary_pointer: Option<usize>,
+    pub interaction_transform: Affine2,
+    pub birthframe: usize,
+}
+
+impl Default for OverlayState {
+    fn default() -> Self {
+        OverlayState {
+            id: AUTO_INCREMENT.fetch_add(1, Ordering::Relaxed),
+            name: Arc::from(""),
+            want_visible: false,
+            show_hide: false,
+            grabbable: false,
+            recenter: false,
+            interactable: false,
+            dirty: true,
+            alpha: 1.0,
+            relative_to: RelativeTo::None,
+            curvature: None,
+            saved_point: None,
+            saved_scale: None,
+            spawn_scale: 1.0,
+            spawn_point: Vec3A::NEG_Z,
+            spawn_rotation: Quat::IDENTITY,
+            transform: Affine3A::IDENTITY,
+            primary_pointer: None,
+            interaction_transform: Affine2::IDENTITY,
+            birthframe: 0,
+        }
+    }
+}
+
+pub struct OverlayData<T>
+where
+    T: Default,
+{
+    pub state: OverlayState,
+    pub backend: Box<dyn OverlayBackend>,
+    pub primary_pointer: Option<usize>,
+    pub data: T,
+}
+
+impl<T> Default for OverlayData<T>
+where
+    T: Default,
+{
+    fn default() -> Self {
+        OverlayData {
+            state: Default::default(),
+            backend: Box::<SplitOverlayBackend>::default(),
+            primary_pointer: None,
+            data: Default::default(),
+        }
+    }
+}
+
+impl OverlayState {
+    pub fn parent_transform(&self, app: &AppState) -> Option<Affine3A> {
+        match self.relative_to {
+            RelativeTo::None => None,
+            RelativeTo::Head => Some(app.input_state.hmd),
+            RelativeTo::Hand(idx) => Some(app.input_state.pointers[idx].pose),
+        }
+    }
+
+    pub fn auto_movement(&mut self, app: &mut AppState) {
+        if let Some(parent) = self.parent_transform(app) {
+            let scale = self.saved_scale.unwrap_or(self.spawn_scale);
+            let point = self.saved_point.unwrap_or(self.spawn_point);
+            self.transform = parent
+                * Affine3A::from_scale_rotation_translation(
+                    Vec3::ONE * scale,
+                    self.spawn_rotation,
+                    point.into(),
+                );
+
+            self.dirty = true;
+        }
+    }
+
+    pub fn reset(&mut self, app: &mut AppState, hard_reset: bool) {
+        if hard_reset {
+            self.saved_point = None;
+            self.saved_scale = None;
+        }
+
+        let scale = self.saved_scale.unwrap_or(self.spawn_scale);
+        let point = self.saved_point.unwrap_or(self.spawn_point);
+
+        let translation = app.input_state.hmd.transform_point3a(point);
+        self.transform = Affine3A::from_scale_rotation_translation(
+            Vec3::ONE * scale,
+            Quat::IDENTITY,
+            translation.into(),
+        );
+        self.realign(&app.input_state.hmd);
+        self.dirty = true;
+    }
+
+    pub fn realign(&mut self, hmd: &Affine3A) {
+        let to_hmd = hmd.translation - self.transform.translation;
+        let up_dir: Vec3A;
+
+        if hmd.x_axis.dot(Vec3A::Y).abs() > 0.2 {
+            // Snap upright
+            up_dir = hmd.y_axis;
+        } else {
+            let dot = to_hmd.normalize().dot(hmd.z_axis);
+            let z_dist = to_hmd.length();
+            let y_dist = (self.transform.translation.y - hmd.translation.y).abs();
+            let x_angle = (y_dist / z_dist).asin();
+
+            if dot < -f32::EPSILON {
+                // facing down
+                let up_point = hmd.translation + z_dist / x_angle.cos() * Vec3A::Y;
+                up_dir = (up_point - self.transform.translation).normalize();
+            } else if dot > f32::EPSILON {
+                // facing up
+                let dn_point = hmd.translation + z_dist / x_angle.cos() * Vec3A::NEG_Y;
+                up_dir = (self.transform.translation - dn_point).normalize();
+            } else {
+                // perfectly upright
+                up_dir = Vec3A::Y;
+            }
+        }
+
+        let scale = self.transform.x_axis.length();
+        self.saved_scale = Some(scale);
+
+        let col_z = (self.transform.translation - hmd.translation).normalize();
+        let col_y = up_dir;
+        let col_x = col_y.cross(col_z);
+        let col_y = col_z.cross(col_x).normalize();
+        let col_x = col_x.normalize();
+
+        let rot = Mat3A::from_quat(self.spawn_rotation)
+            * Mat3A::from_quat(Quat::from_axis_angle(Vec3::Y, PI));
+        self.transform.matrix3 = Mat3A::from_cols(col_x, col_y, col_z).mul_scalar(scale) * rot;
+    }
+}
+
+impl<T> OverlayData<T>
+where
+    T: Default,
+{
+    pub fn init(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        self.state.reset(app, true);
+        self.backend.init(app)
+    }
+    pub fn render(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        self.backend.render(app)
+    }
+    pub fn view(&mut self) -> Option<Arc<ImageView>> {
+        self.backend.view()
+    }
+    pub fn set_visible(&mut self, app: &mut AppState, visible: bool) -> anyhow::Result<()> {
+        let old_visible = self.state.want_visible;
+        self.state.want_visible = visible;
+        if visible != old_visible {
+            if visible {
+                self.backend.resume(app)?;
+            } else {
+                self.backend.pause(app)?;
+            }
+        }
+        Ok(())
+    }
+}
+
+pub trait OverlayRenderer {
+    fn init(&mut self, app: &mut AppState) -> anyhow::Result<()>;
+    fn pause(&mut self, app: &mut AppState) -> anyhow::Result<()>;
+    fn resume(&mut self, app: &mut AppState) -> anyhow::Result<()>;
+    fn render(&mut self, app: &mut AppState) -> anyhow::Result<()>;
+    fn view(&mut self) -> Option<Arc<ImageView>>;
+}
+
+pub struct FallbackRenderer;
+
+impl OverlayRenderer for FallbackRenderer {
+    fn init(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+    fn pause(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+    fn resume(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+    fn render(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+    fn view(&mut self) -> Option<Arc<ImageView>> {
+        None
+    }
+}
+// Boilerplate and dummies
+
+#[derive(Clone, Copy, Debug, Default)]
+pub enum RelativeTo {
+    #[default]
+    None,
+    Head,
+    Hand(usize),
+}
+
+pub struct SplitOverlayBackend {
+    pub renderer: Box<dyn OverlayRenderer>,
+    pub interaction: Box<dyn InteractionHandler>,
+}
+
+impl Default for SplitOverlayBackend {
+    fn default() -> SplitOverlayBackend {
+        SplitOverlayBackend {
+            renderer: Box::new(FallbackRenderer),
+            interaction: Box::new(DummyInteractionHandler),
+        }
+    }
+}
+
+impl OverlayBackend for SplitOverlayBackend {}
+impl OverlayRenderer for SplitOverlayBackend {
+    fn init(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        self.renderer.init(app)
+    }
+    fn pause(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        self.renderer.pause(app)
+    }
+    fn resume(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        self.renderer.resume(app)
+    }
+    fn render(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        self.renderer.render(app)
+    }
+    fn view(&mut self) -> Option<Arc<ImageView>> {
+        self.renderer.view()
+    }
+}
+impl InteractionHandler for SplitOverlayBackend {
+    fn on_left(&mut self, app: &mut AppState, pointer: usize) {
+        self.interaction.on_left(app, pointer);
+    }
+    fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit) -> Option<Haptics> {
+        self.interaction.on_hover(app, hit)
+    }
+    fn on_scroll(&mut self, app: &mut AppState, hit: &PointerHit, delta: f32) {
+        self.interaction.on_scroll(app, hit, delta);
+    }
+    fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool) {
+        self.interaction.on_pointer(app, hit, pressed);
+    }
+}
+
+pub fn ui_transform(extent: &[u32; 2]) -> Affine2 {
+    let aspect = extent[0] as f32 / extent[1] as f32;
+    let scale = if aspect < 1.0 {
+        Vec2 {
+            x: 1.0 / aspect,
+            y: -1.0,
+        }
+    } else {
+        Vec2 {
+            x: 1.0,
+            y: -1.0 * aspect,
+        }
+    };
+    let center = Vec2 { x: 0.5, y: 0.5 };
+    Affine2::from_scale_angle_translation(scale, 0.0, center)
+}

src/config.rs

@@ -0,0 +1,258 @@
+use std::sync::Arc;
+
+use crate::config_io;
+use crate::config_io::get_conf_d_path;
+use crate::gui::modular::ModularUiConfig;
+use crate::load_with_fallback;
+use crate::state::LeftRight;
+use anyhow::bail;
+use log::error;
+use serde::Deserialize;
+use serde::Serialize;
+
+pub fn def_watch_pos() -> [f32; 3] {
+    [-0.03, -0.01, 0.125]
+}
+
+pub fn def_watch_rot() -> [f32; 4] {
+    [-0.7071066, 0.0007963618, 0.7071066, 0.0]
+}
+
+pub fn def_left() -> LeftRight {
+    LeftRight::Left
+}
+
+pub fn def_pw_tokens() -> Vec<(String, String)> {
+    Vec::new()
+}
+
+fn def_click_freeze_time_ms() -> u32 {
+    300
+}
+
+pub fn def_true() -> bool {
+    true
+}
+
+fn def_false() -> bool {
+    false
+}
+
+fn def_one() -> f32 {
+    1.0
+}
+
+pub fn def_half() -> f32 {
+    0.5
+}
+
+pub fn def_point7() -> f32 {
+    0.7
+}
+
+fn def_osc_port() -> u16 {
+    9000
+}
+
+fn def_screens() -> Vec<Arc<str>> {
+    vec![]
+}
+
+fn def_auto() -> Arc<str> {
+    "auto".into()
+}
+
+#[derive(Deserialize, Serialize)]
+pub struct GeneralConfig {
+    #[serde(default = "def_watch_pos")]
+    pub watch_pos: [f32; 3],
+
+    #[serde(default = "def_watch_rot")]
+    pub watch_rot: [f32; 4],
+
+    #[serde(default = "def_left")]
+    pub watch_hand: LeftRight,
+
+    #[serde(default = "def_click_freeze_time_ms")]
+    pub click_freeze_time_ms: u32,
+
+    #[serde(default = "def_true")]
+    pub notifications_enabled: bool,
+
+    #[serde(default = "def_true")]
+    pub notifications_sound_enabled: bool,
+
+    #[serde(default = "def_true")]
+    pub keyboard_sound_enabled: bool,
+
+    #[serde(default = "def_one")]
+    pub keyboard_scale: f32,
+
+    #[serde(default = "def_one")]
+    pub desktop_view_scale: f32,
+
+    #[serde(default = "def_half")]
+    pub watch_view_angle_min: f32,
+
+    #[serde(default = "def_point7")]
+    pub watch_view_angle_max: f32,
+
+    #[serde(default = "def_one")]
+    pub long_press_duration: f32,
+
+    #[serde(default = "def_pw_tokens")]
+    pub pw_tokens: Vec<(String, String)>,
+
+    #[serde(default = "def_osc_port")]
+    pub osc_out_port: u16,
+
+    #[serde(default = "def_false")]
+    pub upright_screen_fix: bool,
+
+    #[serde(default = "def_false")]
+    pub double_cursor_fix: bool,
+
+    #[serde(default = "def_screens")]
+    pub show_screens: Vec<Arc<str>>,
+
+    #[serde(default = "def_auto")]
+    pub capture_method: Arc<str>,
+
+    #[serde(default = "def_point7")]
+    pub xr_grab_sensitivity: f32,
+
+    #[serde(default = "def_point7")]
+    pub xr_click_sensitivity: f32,
+
+    #[serde(default = "def_point7")]
+    pub xr_alt_click_sensitivity: f32,
+
+    #[serde(default = "def_half")]
+    pub xr_grab_sensitivity_release: f32,
+
+    #[serde(default = "def_half")]
+    pub xr_click_sensitivity_release: f32,
+
+    #[serde(default = "def_half")]
+    pub xr_alt_click_sensitivity_release: f32,
+}
+
+impl GeneralConfig {
+    fn sanitize_range(name: &str, val: f32, from: f32, to: f32) {
+        if !val.is_normal() || val < from || val > to {
+            panic!(
+                "GeneralConfig: {} needs to be between {} and {}",
+                name, from, to
+            );
+        }
+    }
+
+    pub fn load_from_disk() -> GeneralConfig {
+        let config = load_general();
+        config.post_load();
+        config
+    }
+
+    fn post_load(&self) {
+        GeneralConfig::sanitize_range("keyboard_scale", self.keyboard_scale, 0.05, 5.0);
+        GeneralConfig::sanitize_range("desktop_view_scale", self.desktop_view_scale, 0.05, 5.0);
+    }
+}
+
+const FALLBACKS: [&str; 3] = [
+    include_str!("res/keyboard.yaml"),
+    include_str!("res/watch.yaml"),
+    include_str!("res/settings.yaml"),
+];
+
+const FILES: [&str; 3] = ["keyboard.yaml", "watch.yaml", "settings.yaml"];
+
+#[derive(Clone, Copy)]
+#[repr(usize)]
+pub enum ConfigType {
+    Keyboard,
+    Watch,
+    Settings,
+}
+
+pub fn load_known_yaml<T>(config_type: ConfigType) -> T
+where
+    T: for<'de> Deserialize<'de>,
+{
+    let fallback = FALLBACKS[config_type as usize];
+    let file_name = FILES[config_type as usize];
+    let maybe_override = config_io::load(file_name);
+
+    for yaml in [maybe_override.as_deref(), Some(fallback)].iter().flatten() {
+        match serde_yaml::from_str::<T>(yaml) {
+            Ok(d) => return d,
+            Err(e) => {
+                error!("Failed to parse {}, falling back to defaults.", file_name);
+                error!("{}", e);
+            }
+        }
+    }
+    // can only get here if internal fallback is broken
+    panic!("No usable config found.");
+}
+
+pub fn load_custom_ui(name: &str) -> anyhow::Result<ModularUiConfig> {
+    let filename = format!("{}.yaml", name);
+    let Some(yaml_data) = config_io::load(&filename) else {
+        bail!("Could not read file at {}", &filename);
+    };
+    Ok(serde_yaml::from_str(&yaml_data)?)
+}
+
+pub fn load_general() -> GeneralConfig {
+    let mut yaml_data = String::new();
+
+    // Add files from conf.d directory
+    let path_conf_d = get_conf_d_path();
+    if let Ok(paths_unsorted) = std::fs::read_dir(path_conf_d) {
+        let mut paths: Vec<_> = paths_unsorted
+            .filter_map(|r| match r {
+                Ok(entry) => Some(entry),
+                Err(e) => {
+                    error!("Failed to read conf.d directory: {}", e);
+                    None
+                }
+            })
+            .collect();
+        // Sort paths alphabetically
+        paths.sort_by_key(|dir| dir.path());
+        for path in paths {
+            let file_type = match path.file_type() {
+                Ok(file_type) => file_type,
+                Err(e) => {
+                    error!(
+                        "Failed to get file type of {}: {}",
+                        path.path().to_string_lossy(),
+                        e
+                    );
+                    continue;
+                }
+            };
+
+            if !file_type.is_file() {
+                continue;
+            }
+
+            log::info!("Loading config file {}", path.path().to_string_lossy());
+
+            if let Ok(data) = std::fs::read_to_string(path.path()) {
+                yaml_data.push('\n'); // Just in case, if end of the config file was not newline
+                yaml_data.push_str(data.as_str());
+            } else {
+                // Shouldn't happen anyways
+                error!("Failed to load {}", path.path().to_string_lossy());
+            }
+        }
+    }
+
+    if yaml_data.is_empty() {
+        yaml_data.push_str(load_with_fallback!("config.yaml", "res/config.yaml").as_str());
+    }
+
+    serde_yaml::from_str(&yaml_data).expect("Failed to parse config.yaml")
+}

src/config_io.rs

@@ -0,0 +1,71 @@
+use log::error;
+use once_cell::sync::Lazy;
+use std::{
+    fs::{self, create_dir},
+    path::PathBuf,
+};
+
+const FALLBACK_CONFIG_PATH: &str = "/tmp/wlxoverlay";
+
+pub static CONFIG_ROOT_PATH: Lazy<PathBuf> = Lazy::new(|| {
+    if let Ok(xdg_dirs) = xdg::BaseDirectories::new() {
+        let mut dir = xdg_dirs.get_config_home();
+        dir.push("wlxoverlay");
+        return dir;
+    }
+    //Return fallback config path
+    error!(
+        "Err: Failed to find config path, using {}",
+        FALLBACK_CONFIG_PATH
+    );
+    PathBuf::from(FALLBACK_CONFIG_PATH)
+});
+
+pub fn get_conf_d_path() -> PathBuf {
+    let mut config_root = CONFIG_ROOT_PATH.clone();
+    config_root.push("conf.d");
+    config_root
+}
+
+// Make sure config directory is present and return root config path
+pub fn ensure_config_root() -> PathBuf {
+    let path = CONFIG_ROOT_PATH.clone();
+    let _ = create_dir(&path);
+
+    let path_conf_d = get_conf_d_path();
+    let _ = create_dir(path_conf_d);
+    path
+}
+
+fn get_config_file_path(filename: &str) -> PathBuf {
+    let mut config_root = CONFIG_ROOT_PATH.clone();
+    config_root.push(filename);
+    config_root
+}
+
+pub fn load(filename: &str) -> Option<String> {
+    let path = get_config_file_path(filename);
+    log::info!("Loading config {}", path.to_string_lossy());
+
+    if let Ok(data) = fs::read_to_string(path) {
+        Some(data)
+    } else {
+        None
+    }
+}
+
+#[macro_export]
+macro_rules! load_with_fallback {
+    ($filename: expr,  $fallback: expr) => {
+        if let Some(data) = config_io::load($filename) {
+            data
+        } else {
+            log::info!(
+                "Config {}/{} does not exist, using internal fallback",
+                config_io::CONFIG_ROOT_PATH.to_string_lossy(),
+                $filename
+            );
+            include_str!($fallback).to_string()
+        }
+    };
+}

src/gui/font.rs

@@ -0,0 +1,227 @@
+use std::{rc::Rc, str::FromStr, sync::Arc};
+
+use fontconfig::{FontConfig, OwnedPattern};
+use freetype::{bitmap::PixelMode, face::LoadFlag, Face, Library};
+use idmap::IdMap;
+use vulkano::{command_buffer::CommandBufferUsage, format::Format, image::Image};
+
+use crate::graphics::WlxGraphics;
+
+const PRIMARY_FONT: &str = "LiberationSans";
+
+pub struct FontCache {
+    fc: FontConfig,
+    ft: Library,
+    collections: IdMap<isize, FontCollection>,
+}
+
+struct FontCollection {
+    fonts: Vec<Font>,
+    cp_map: IdMap<usize, usize>,
+}
+
+struct Font {
+    face: Face,
+    glyphs: IdMap<usize, Rc<Glyph>>,
+}
+
+pub struct Glyph {
+    pub tex: Option<Arc<Image>>,
+    pub top: f32,
+    pub left: f32,
+    pub width: f32,
+    pub height: f32,
+    pub advance: f32,
+}
+
+impl FontCache {
+    pub fn new() -> anyhow::Result<Self> {
+        let ft = Library::init()?;
+        let fc = FontConfig::default();
+
+        Ok(FontCache {
+            fc,
+            ft,
+            collections: IdMap::new(),
+        })
+    }
+
+    pub fn get_text_size(
+        &mut self,
+        text: &str,
+        size: isize,
+        graphics: Arc<WlxGraphics>,
+    ) -> anyhow::Result<(f32, f32)> {
+        let sizef = size as f32;
+
+        let height = sizef + ((text.lines().count() as f32) - 1f32) * (sizef * 1.5);
+
+        let mut max_w = sizef * 0.33;
+        for line in text.lines() {
+            let w: f32 = line
+                .chars()
+                .filter_map(|c| {
+                    self.get_glyph_for_cp(c as usize, size, graphics.clone())
+                        .map(|glyph| glyph.advance)
+                        .ok()
+                })
+                .sum();
+
+            if w > max_w {
+                max_w = w;
+            }
+        }
+        Ok((max_w, height))
+    }
+
+    pub fn get_glyphs(
+        &mut self,
+        text: &str,
+        size: isize,
+        graphics: Arc<WlxGraphics>,
+    ) -> anyhow::Result<Vec<Rc<Glyph>>> {
+        let mut glyphs = Vec::new();
+        for line in text.lines() {
+            for c in line.chars() {
+                glyphs.push(self.get_glyph_for_cp(c as usize, size, graphics.clone())?);
+            }
+        }
+        Ok(glyphs)
+    }
+
+    fn get_font_for_cp(&mut self, cp: usize, size: isize) -> usize {
+        if !self.collections.contains_key(size) {
+            self.collections.insert(
+                size,
+                FontCollection {
+                    fonts: Vec::new(),
+                    cp_map: IdMap::new(),
+                },
+            );
+        }
+        let coll = self.collections.get_mut(size).unwrap(); // safe because of the insert above
+
+        if let Some(font) = coll.cp_map.get(cp) {
+            return *font;
+        }
+
+        let pattern_str = format!("{PRIMARY_FONT}-{size}:style=bold:charset={cp:04x}");
+        let mut pattern = OwnedPattern::from_str(&pattern_str).unwrap(); // safe because PRIMARY_FONT is const
+        self.fc
+            .substitute(&mut pattern, fontconfig::MatchKind::Pattern);
+        pattern.default_substitute();
+
+        let pattern = pattern.font_match(&mut self.fc);
+
+        if let Some(path) = pattern.filename() {
+            log::debug!(
+                "Loading font: {} {}pt",
+                pattern.name().unwrap_or(path),
+                size
+            );
+
+            let font_idx = pattern.face_index().unwrap_or(0);
+
+            let face = match self.ft.new_face(path, font_idx as _) {
+                Ok(face) => face,
+                Err(e) => {
+                    log::warn!("Failed to load font at {}: {:?}", path, e);
+                    coll.cp_map.insert(cp, 0);
+                    return 0;
+                }
+            };
+            match face.set_char_size(size << 6, size << 6, 96, 96) {
+                Ok(_) => {}
+                Err(e) => {
+                    log::warn!("Failed to set font size: {:?}", e);
+                    coll.cp_map.insert(cp, 0);
+                    return 0;
+                }
+            };
+
+            let idx = coll.fonts.len();
+            for cp in 0..0xFFFF {
+                if coll.cp_map.contains_key(cp) {
+                    continue;
+                }
+                let g = face.get_char_index(cp);
+                if g > 0 {
+                    coll.cp_map.insert(cp, idx);
+                }
+            }
+
+            let zero_glyph = Rc::new(Glyph {
+                tex: None,
+                top: 0.,
+                left: 0.,
+                width: 0.,
+                height: 0.,
+                advance: size as f32 / 3.,
+            });
+            let mut glyphs = IdMap::new();
+            glyphs.insert(0, zero_glyph);
+
+            let font = Font { face, glyphs };
+            coll.fonts.push(font);
+
+            return idx;
+        }
+        coll.cp_map.insert(cp, 0);
+        0
+    }
+
+    fn get_glyph_for_cp(
+        &mut self,
+        cp: usize,
+        size: isize,
+        graphics: Arc<WlxGraphics>,
+    ) -> anyhow::Result<Rc<Glyph>> {
+        let key = self.get_font_for_cp(cp, size);
+
+        let font = &mut self.collections[size].fonts[key];
+
+        if let Some(glyph) = font.glyphs.get(cp) {
+            return Ok(glyph.clone());
+        }
+
+        if font.face.load_char(cp, LoadFlag::DEFAULT).is_err() {
+            return Ok(font.glyphs[0].clone());
+        }
+
+        let glyph = font.face.glyph();
+        if glyph.render_glyph(freetype::RenderMode::Normal).is_err() {
+            return Ok(font.glyphs[0].clone());
+        }
+
+        let bmp = glyph.bitmap();
+        let buf = bmp.buffer().to_vec();
+        if buf.is_empty() {
+            return Ok(font.glyphs[0].clone());
+        }
+
+        let metrics = glyph.metrics();
+
+        let format = match bmp.pixel_mode() {
+            Ok(PixelMode::Gray) => Format::R8_UNORM,
+            Ok(PixelMode::Gray2) => Format::R16_SFLOAT,
+            Ok(PixelMode::Gray4) => Format::R32_SFLOAT,
+            _ => return Ok(font.glyphs[0].clone()),
+        };
+
+        let mut cmd_buffer = graphics.create_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+        let texture = cmd_buffer.texture2d(bmp.width() as _, bmp.rows() as _, format, &buf)?;
+        cmd_buffer.build_and_execute_now()?;
+
+        let g = Glyph {
+            tex: Some(texture),
+            top: (metrics.horiBearingY >> 6i64) as _,
+            left: (metrics.horiBearingX >> 6i64) as _,
+            advance: (metrics.horiAdvance >> 6i64) as _,
+            width: bmp.width() as _,
+            height: bmp.rows() as _,
+        };
+
+        font.glyphs.insert(cp, Rc::new(g));
+        Ok(font.glyphs[cp].clone())
+    }
+}

src/gui/mod.rs

@@ -0,0 +1,743 @@
+use std::sync::Arc;
+
+use anyhow::bail;
+use glam::{Vec2, Vec3, Vec4};
+use vulkano::{
+    command_buffer::CommandBufferUsage,
+    format::Format,
+    image::{sampler::Filter, view::ImageView, ImageLayout},
+};
+
+use crate::{
+    backend::{
+        input::{Haptics, InteractionHandler, PointerHit, PointerMode},
+        overlay::{OverlayBackend, OverlayRenderer},
+    },
+    graphics::{
+        WlxCommandBuffer, WlxGraphics, WlxPass, WlxPipeline, WlxPipelineLegacy, BLEND_ALPHA,
+    },
+    state::AppState,
+};
+
+pub mod font;
+pub mod modular;
+
+const RES_DIVIDER: usize = 4;
+
+struct Rect {
+    x: f32,
+    y: f32,
+    w: f32,
+    h: f32,
+}
+
+// Parses a color from a HTML hex string
+pub fn color_parse(html_hex: &str) -> anyhow::Result<Vec3> {
+    if html_hex.len() == 7 {
+        if let (Ok(r), Ok(g), Ok(b)) = (
+            u8::from_str_radix(&html_hex[1..3], 16),
+            u8::from_str_radix(&html_hex[3..5], 16),
+            u8::from_str_radix(&html_hex[5..7], 16),
+        ) {
+            return Ok(Vec3::new(r as f32 / 255., g as f32 / 255., b as f32 / 255.));
+        }
+    }
+    bail!(
+        "Invalid color string: '{}', must be 7 characters long (e.g. #FF00FF)",
+        &html_hex
+    )
+}
+
+pub struct CanvasBuilder<D, S> {
+    canvas: Canvas<D, S>,
+
+    pub fg_color: Vec3,
+    pub bg_color: Vec3,
+    pub font_size: isize,
+}
+
+impl<D, S> CanvasBuilder<D, S> {
+    pub fn new(
+        width: usize,
+        height: usize,
+        graphics: Arc<WlxGraphics>,
+        format: Format,
+        data: D,
+    ) -> anyhow::Result<Self> {
+        Ok(Self {
+            canvas: Canvas::new(width, height, graphics, format, data)?,
+            bg_color: Vec3::ZERO,
+            fg_color: Vec3::ONE,
+            font_size: 16,
+        })
+    }
+
+    pub fn build(self) -> Canvas<D, S> {
+        self.canvas
+    }
+
+    // Creates a panel with bg_color inherited from the canvas
+    pub fn panel(&mut self, x: f32, y: f32, w: f32, h: f32) -> &mut Control<D, S> {
+        let idx = self.canvas.controls.len();
+        self.canvas.controls.push(Control {
+            rect: Rect { x, y, w, h },
+            bg_color: self.bg_color,
+            on_render_bg: Some(Control::render_rect),
+            ..Control::new()
+        });
+        &mut self.canvas.controls[idx]
+    }
+
+    // Creates a label with fg_color, font_size inherited from the canvas
+    pub fn label(&mut self, x: f32, y: f32, w: f32, h: f32, text: Arc<str>) -> &mut Control<D, S> {
+        let idx = self.canvas.controls.len();
+        self.canvas.controls.push(Control {
+            rect: Rect { x, y, w, h },
+            text,
+            fg_color: self.fg_color,
+            size: self.font_size,
+            on_render_fg: Some(Control::render_text),
+            ..Control::new()
+        });
+        &mut self.canvas.controls[idx]
+    }
+
+    // Creates a label with fg_color, font_size inherited from the canvas
+    #[allow(dead_code)]
+    pub fn label_centered(
+        &mut self,
+        x: f32,
+        y: f32,
+        w: f32,
+        h: f32,
+        text: Arc<str>,
+    ) -> &mut Control<D, S> {
+        let idx = self.canvas.controls.len();
+        self.canvas.controls.push(Control {
+            rect: Rect { x, y, w, h },
+            text,
+            fg_color: self.fg_color,
+            size: self.font_size,
+            on_render_fg: Some(Control::render_text_centered),
+            ..Control::new()
+        });
+        &mut self.canvas.controls[idx]
+    }
+
+    // Creates a button with fg_color, bg_color, font_size inherited from the canvas
+    pub fn button(&mut self, x: f32, y: f32, w: f32, h: f32, text: Arc<str>) -> &mut Control<D, S> {
+        let idx = self.canvas.controls.len();
+
+        self.canvas.interactive_set_idx(x, y, w, h, idx);
+        self.canvas.controls.push(Control {
+            rect: Rect { x, y, w, h },
+            text,
+            fg_color: self.fg_color,
+            bg_color: self.bg_color,
+            size: self.font_size,
+            on_render_bg: Some(Control::render_rect),
+            on_render_fg: Some(Control::render_text_centered),
+            on_render_hl: Some(Control::render_highlight),
+            ..Control::new()
+        });
+
+        &mut self.canvas.controls[idx]
+    }
+
+    pub fn key_button(
+        &mut self,
+        x: f32,
+        y: f32,
+        w: f32,
+        h: f32,
+        label: &[String],
+    ) -> &mut Control<D, S> {
+        let idx = self.canvas.controls.len();
+        self.canvas.interactive_set_idx(x, y, w, h, idx);
+
+        self.canvas.controls.push(Control {
+            rect: Rect { x, y, w, h },
+            bg_color: self.bg_color,
+            on_render_bg: Some(Control::render_rect),
+            on_render_hl: Some(Control::render_highlight),
+            ..Control::new()
+        });
+
+        for (i, item) in label.iter().enumerate().take(label.len().min(2)) {
+            self.canvas.controls.push(Control {
+                rect: if i == 0 {
+                    Rect {
+                        x: x + 4.,
+                        y: y + (self.font_size as f32) + 4.,
+                        w,
+                        h,
+                    }
+                } else {
+                    Rect {
+                        x: x + w * 0.5,
+                        y: y + h - (self.font_size as f32) + 4.,
+                        w,
+                        h,
+                    }
+                },
+                text: Arc::from(item.as_str()),
+                fg_color: self.fg_color,
+                size: self.font_size,
+                on_render_fg: Some(Control::render_text),
+                ..Control::new()
+            });
+        }
+
+        &mut self.canvas.controls[idx]
+    }
+}
+
+pub struct CanvasData<D> {
+    pub data: D,
+    pub width: usize,
+    pub height: usize,
+
+    graphics: Arc<WlxGraphics>,
+
+    pipeline_bg_color: Arc<WlxPipeline<WlxPipelineLegacy>>,
+    pipeline_fg_glyph: Arc<WlxPipeline<WlxPipelineLegacy>>,
+    pipeline_final: Arc<WlxPipeline<WlxPipelineLegacy>>,
+}
+
+pub struct Canvas<D, S> {
+    controls: Vec<Control<D, S>>,
+    canvas: CanvasData<D>,
+
+    hover_controls: [Option<usize>; 2],
+    pressed_controls: [Option<usize>; 2],
+
+    interact_map: Vec<Option<u8>>,
+    interact_stride: usize,
+    interact_rows: usize,
+
+    view_final: Arc<ImageView>,
+
+    pass_fg: WlxPass<WlxPipelineLegacy>,
+    pass_bg: WlxPass<WlxPipelineLegacy>,
+}
+
+impl<D, S> Canvas<D, S> {
+    fn new(
+        width: usize,
+        height: usize,
+        graphics: Arc<WlxGraphics>,
+        format: Format,
+        data: D,
+    ) -> anyhow::Result<Self> {
+        let tex_fg = graphics.render_texture(width as _, height as _, format)?;
+        let tex_bg = graphics.render_texture(width as _, height as _, format)?;
+        let tex_final = graphics.render_texture(width as _, height as _, format)?;
+
+        let view_fg = ImageView::new_default(tex_fg.clone())?;
+        let view_bg = ImageView::new_default(tex_bg.clone())?;
+        let view_final = ImageView::new_default(tex_final.clone())?;
+
+        let Ok(shaders) = graphics.shared_shaders.read() else {
+            bail!("Failed to lock shared shaders for reading");
+        };
+
+        let pipeline_bg_color = graphics.create_pipeline(
+            view_bg.clone(),
+            shaders.get("vert_common").unwrap().clone(), // want panic
+            shaders.get("frag_color").unwrap().clone(),  // want panic
+            format,
+            Some(BLEND_ALPHA),
+        )?;
+
+        let pipeline_fg_glyph = graphics.create_pipeline(
+            view_fg.clone(),
+            shaders.get("vert_common").unwrap().clone(), // want panic
+            shaders.get("frag_glyph").unwrap().clone(),  // want panic
+            format,
+            Some(BLEND_ALPHA),
+        )?;
+
+        let vertex_buffer =
+            graphics.upload_verts(width as _, height as _, 0., 0., width as _, height as _)?;
+
+        let pipeline_final = graphics.create_pipeline_with_layouts(
+            view_final.clone(),
+            shaders.get("vert_common").unwrap().clone(), // want panic
+            shaders.get("frag_sprite").unwrap().clone(), // want panic
+            format,
+            Some(BLEND_ALPHA),
+            ImageLayout::TransferSrcOptimal,
+            ImageLayout::TransferSrcOptimal,
+        )?;
+
+        let set_fg = pipeline_final.uniform_sampler(0, view_fg.clone(), Filter::Linear)?;
+        let set_bg = pipeline_final.uniform_sampler(0, view_bg.clone(), Filter::Linear)?;
+        let pass_fg = pipeline_final.create_pass(
+            [width as _, height as _],
+            vertex_buffer.clone(),
+            graphics.quad_indices.clone(),
+            vec![set_fg],
+        )?;
+        let pass_bg = pipeline_final.create_pass(
+            [width as _, height as _],
+            vertex_buffer.clone(),
+            graphics.quad_indices.clone(),
+            vec![set_bg],
+        )?;
+
+        let stride = width / RES_DIVIDER;
+        let rows = height / RES_DIVIDER;
+
+        Ok(Self {
+            canvas: CanvasData {
+                data,
+                width,
+                height,
+                graphics: graphics.clone(),
+                pipeline_bg_color,
+                pipeline_fg_glyph,
+                pipeline_final,
+            },
+            controls: Vec::new(),
+            hover_controls: [None, None],
+            pressed_controls: [None, None],
+            interact_map: vec![None; stride * rows],
+            interact_stride: stride,
+            interact_rows: rows,
+            view_final,
+            pass_fg,
+            pass_bg,
+        })
+    }
+
+    fn interactive_set_idx(&mut self, x: f32, y: f32, w: f32, h: f32, idx: usize) {
+        let (x, y, w, h) = (x as usize, y as usize, w as usize, h as usize);
+
+        let x_min = (x / RES_DIVIDER).max(0);
+        let y_min = (y / RES_DIVIDER).max(0);
+        let x_max = (x_min + (w / RES_DIVIDER)).min(self.interact_stride - 1);
+        let y_max = (y_min + (h / RES_DIVIDER)).min(self.interact_rows - 1);
+
+        for y in y_min..y_max {
+            for x in x_min..x_max {
+                self.interact_map[y * self.interact_stride + x] = Some(idx as u8);
+            }
+        }
+    }
+
+    fn interactive_get_idx(&self, uv: Vec2) -> Option<usize> {
+        let x = (uv.x * self.canvas.width as f32) as usize;
+        let y = (uv.y * self.canvas.height as f32) as usize;
+        let x = (x / RES_DIVIDER).max(0).min(self.interact_stride - 1);
+        let y = (y / RES_DIVIDER).max(0).min(self.interact_rows - 1);
+        self.interact_map[y * self.interact_stride + x].map(|x| x as usize)
+    }
+
+    fn render_bg(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        let mut cmd_buffer = self
+            .canvas
+            .graphics
+            .create_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+        cmd_buffer.begin_render_pass(&self.canvas.pipeline_bg_color)?;
+        for c in self.controls.iter_mut() {
+            if let Some(fun) = c.on_render_bg {
+                fun(c, &self.canvas, app, &mut cmd_buffer)?;
+            }
+        }
+        cmd_buffer.end_render_pass()?;
+        cmd_buffer.build_and_execute_now()
+    }
+
+    fn render_fg(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        let mut cmd_buffer = self
+            .canvas
+            .graphics
+            .create_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+        cmd_buffer.begin_render_pass(&self.canvas.pipeline_fg_glyph)?;
+        for c in self.controls.iter_mut() {
+            if let Some(fun) = c.on_render_fg {
+                fun(c, &self.canvas, app, &mut cmd_buffer)?;
+            }
+        }
+        cmd_buffer.end_render_pass()?;
+        cmd_buffer.build_and_execute_now()
+    }
+}
+
+impl<D, S> InteractionHandler for Canvas<D, S> {
+    fn on_left(&mut self, _app: &mut AppState, pointer: usize) {
+        self.hover_controls[pointer] = None;
+    }
+    fn on_hover(&mut self, _app: &mut AppState, hit: &PointerHit) -> Option<Haptics> {
+        let old = self.hover_controls[hit.pointer];
+        if let Some(i) = self.interactive_get_idx(hit.uv) {
+            self.hover_controls[hit.pointer] = Some(i);
+        } else {
+            self.hover_controls[hit.pointer] = None;
+        }
+        if old != self.hover_controls[hit.pointer] {
+            Some(Haptics {
+                intensity: 0.1,
+                duration: 0.01,
+                frequency: 5.0,
+            })
+        } else {
+            None
+        }
+    }
+    fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool) {
+        let idx = if pressed {
+            self.interactive_get_idx(hit.uv)
+        } else {
+            self.pressed_controls[hit.pointer]
+        };
+
+        if let Some(idx) = idx {
+            let c = &mut self.controls[idx];
+            if pressed {
+                if let Some(ref mut f) = c.on_press {
+                    self.pressed_controls[hit.pointer] = Some(idx);
+                    f(c, &mut self.canvas.data, app, hit.mode);
+                }
+            } else if let Some(ref mut f) = c.on_release {
+                self.pressed_controls[hit.pointer] = None;
+                f(c, &mut self.canvas.data, app);
+            }
+        }
+    }
+    fn on_scroll(&mut self, app: &mut AppState, hit: &PointerHit, delta: f32) {
+        let idx = self.hover_controls[hit.pointer];
+
+        if let Some(idx) = idx {
+            let c = &mut self.controls[idx];
+            if let Some(ref mut f) = c.on_scroll {
+                f(c, &mut self.canvas.data, app, delta);
+            }
+        }
+    }
+}
+
+impl<D, S> OverlayRenderer for Canvas<D, S> {
+    fn init(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        self.render_bg(app)?;
+        self.render_fg(app)
+    }
+    fn pause(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+    fn resume(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+    fn render(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        let mut dirty = false;
+
+        for c in self.controls.iter_mut() {
+            if let Some(fun) = c.on_update {
+                fun(c, &mut self.canvas.data, app);
+            }
+            if c.dirty {
+                dirty = true;
+                c.dirty = false;
+            }
+        }
+
+        if dirty {
+            self.render_bg(app)?;
+            self.render_fg(app)?;
+        }
+
+        /*
+        let image = self.view_final.image().clone();
+        if self.first_render {
+            self.first_render = false;
+        } else {
+            self.canvas
+                .graphics
+                .transition_layout(
+                    image.clone(),
+                    ImageLayout::TransferSrcOptimal,
+                    ImageLayout::ColorAttachmentOptimal,
+                )
+                .wait(None)
+                .unwrap();
+        }
+        */
+
+        let mut cmd_buffer = self
+            .canvas
+            .graphics
+            .create_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+        cmd_buffer.begin_render_pass(&self.canvas.pipeline_final)?;
+
+        // static background
+        cmd_buffer.run_ref(&self.pass_bg)?;
+
+        for (i, c) in self.controls.iter_mut().enumerate() {
+            if let Some(render) = c.on_render_hl {
+                if let Some(test) = c.test_highlight {
+                    if let Some(hl_color) = test(c, &mut self.canvas.data, app) {
+                        render(c, &self.canvas, app, &mut cmd_buffer, hl_color)?;
+                    }
+                }
+                if self.hover_controls.contains(&Some(i)) {
+                    render(
+                        c,
+                        &self.canvas,
+                        app,
+                        &mut cmd_buffer,
+                        Vec4::new(1., 1., 1., 0.3),
+                    )?;
+                }
+            }
+        }
+
+        // mostly static text
+        cmd_buffer.run_ref(&self.pass_fg)?;
+
+        cmd_buffer.end_render_pass()?;
+        cmd_buffer.build_and_execute_now()
+
+        /*
+        self.canvas
+            .graphics
+            .transition_layout(
+                image,
+                ImageLayout::ColorAttachmentOptimal,
+                ImageLayout::TransferSrcOptimal,
+            )
+            .wait(None)
+            .unwrap();
+        */
+    }
+    fn view(&mut self) -> Option<Arc<ImageView>> {
+        Some(self.view_final.clone())
+    }
+}
+
+impl<D, S> OverlayBackend for Canvas<D, S> {}
+
+pub struct Control<D, S> {
+    pub state: Option<S>,
+    rect: Rect,
+    fg_color: Vec3,
+    bg_color: Vec3,
+    text: Arc<str>,
+    size: isize,
+    dirty: bool,
+
+    pub on_update: Option<fn(&mut Self, &mut D, &mut AppState)>,
+    pub on_press: Option<fn(&mut Self, &mut D, &mut AppState, PointerMode)>,
+    pub on_release: Option<fn(&mut Self, &mut D, &mut AppState)>,
+    pub on_scroll: Option<fn(&mut Self, &mut D, &mut AppState, f32)>,
+    pub test_highlight: Option<fn(&Self, &mut D, &mut AppState) -> Option<Vec4>>,
+
+    on_render_bg: Option<
+        fn(&Self, &CanvasData<D>, &mut AppState, &mut WlxCommandBuffer) -> anyhow::Result<()>,
+    >,
+    on_render_hl: Option<
+        fn(&Self, &CanvasData<D>, &mut AppState, &mut WlxCommandBuffer, Vec4) -> anyhow::Result<()>,
+    >,
+    on_render_fg: Option<
+        fn(&Self, &CanvasData<D>, &mut AppState, &mut WlxCommandBuffer) -> anyhow::Result<()>,
+    >,
+}
+
+impl<D, S> Control<D, S> {
+    fn new() -> Self {
+        Self {
+            rect: Rect {
+                x: 0.,
+                y: 0.,
+                w: 0.,
+                h: 0.,
+            },
+            fg_color: Vec3::ONE,
+            bg_color: Vec3::ZERO,
+            text: Arc::from(""),
+            dirty: true,
+            size: 24,
+            state: None,
+            on_update: None,
+            on_render_bg: None,
+            on_render_hl: None,
+            on_render_fg: None,
+            test_highlight: None,
+            on_press: None,
+            on_release: None,
+            on_scroll: None,
+        }
+    }
+
+    #[inline(always)]
+    pub fn set_text(&mut self, text: &str) {
+        if *self.text == *text {
+            return;
+        }
+        self.text = text.into();
+        self.dirty = true;
+    }
+
+    #[inline(always)]
+    pub fn set_fg_color(&mut self, color: Vec3) {
+        if self.fg_color == color {
+            return;
+        }
+        self.fg_color = color;
+        self.dirty = true;
+    }
+
+    fn render_rect(
+        &self,
+        canvas: &CanvasData<D>,
+        _: &mut AppState,
+        cmd_buffer: &mut WlxCommandBuffer,
+    ) -> anyhow::Result<()> {
+        let pass = {
+            let vertex_buffer = canvas.graphics.upload_verts(
+                canvas.width as _,
+                canvas.height as _,
+                self.rect.x,
+                self.rect.y,
+                self.rect.w,
+                self.rect.h,
+            )?;
+            let set0 = canvas.pipeline_bg_color.uniform_buffer(
+                0,
+                vec![self.bg_color.x, self.bg_color.y, self.bg_color.z, 1.],
+            )?;
+            canvas.pipeline_bg_color.create_pass(
+                [canvas.width as _, canvas.height as _],
+                vertex_buffer,
+                canvas.graphics.quad_indices.clone(),
+                vec![set0],
+            )?
+        };
+
+        cmd_buffer.run_ref(&pass)
+    }
+
+    fn render_highlight(
+        &self,
+        canvas: &CanvasData<D>,
+        _: &mut AppState,
+        cmd_buffer: &mut WlxCommandBuffer,
+        color: Vec4,
+    ) -> anyhow::Result<()> {
+        let vertex_buffer = canvas.graphics.upload_verts(
+            canvas.width as _,
+            canvas.height as _,
+            self.rect.x,
+            self.rect.y,
+            self.rect.w,
+            self.rect.h,
+        )?;
+
+        let set0 = canvas
+            .pipeline_bg_color
+            .uniform_buffer(0, color.to_array().to_vec())?;
+
+        let pass = canvas.pipeline_bg_color.create_pass(
+            [canvas.width as _, canvas.height as _],
+            vertex_buffer.clone(),
+            canvas.graphics.quad_indices.clone(),
+            vec![set0],
+        )?;
+
+        cmd_buffer.run_ref(&pass)
+    }
+
+    fn render_text(
+        &self,
+        canvas: &CanvasData<D>,
+        app: &mut AppState,
+        cmd_buffer: &mut WlxCommandBuffer,
+    ) -> anyhow::Result<()> {
+        let mut cur_y = self.rect.y;
+        for line in self.text.lines() {
+            let mut cur_x = self.rect.x;
+            for glyph in app
+                .fc
+                .get_glyphs(line, self.size, canvas.graphics.clone())?
+            {
+                if let Some(tex) = glyph.tex.clone() {
+                    let vertex_buffer = canvas.graphics.upload_verts(
+                        canvas.width as _,
+                        canvas.height as _,
+                        cur_x + glyph.left,
+                        cur_y - glyph.top,
+                        glyph.width,
+                        glyph.height,
+                    )?;
+                    let set0 = canvas.pipeline_fg_glyph.uniform_sampler(
+                        0,
+                        ImageView::new_default(tex)?,
+                        Filter::Linear,
+                    )?;
+                    let set1 = canvas.pipeline_fg_glyph.uniform_buffer(
+                        1,
+                        vec![self.fg_color.x, self.fg_color.y, self.fg_color.z, 1.],
+                    )?;
+                    let pass = canvas.pipeline_fg_glyph.create_pass(
+                        [canvas.width as _, canvas.height as _],
+                        vertex_buffer,
+                        canvas.graphics.quad_indices.clone(),
+                        vec![set0, set1],
+                    )?;
+                    cmd_buffer.run_ref(&pass)?;
+                }
+                cur_x += glyph.advance;
+            }
+            cur_y += (self.size as f32) * 1.5;
+        }
+        Ok(())
+    }
+    fn render_text_centered(
+        &self,
+        canvas: &CanvasData<D>,
+        app: &mut AppState,
+        cmd_buffer: &mut WlxCommandBuffer,
+    ) -> anyhow::Result<()> {
+        let (w, h) = app
+            .fc
+            .get_text_size(&self.text, self.size, canvas.graphics.clone())?;
+
+        let mut cur_y = self.rect.y + (self.rect.h) - (h * 0.5) - (self.size as f32 * 0.25);
+        for line in self.text.lines() {
+            let mut cur_x = self.rect.x + (self.rect.w * 0.5) - (w * 0.5);
+            for glyph in app
+                .fc
+                .get_glyphs(line, self.size, canvas.graphics.clone())?
+            {
+                if let Some(tex) = glyph.tex.clone() {
+                    let vertex_buffer = canvas.graphics.upload_verts(
+                        canvas.width as _,
+                        canvas.height as _,
+                        cur_x + glyph.left,
+                        cur_y - glyph.top,
+                        glyph.width,
+                        glyph.height,
+                    )?;
+                    let set0 = canvas.pipeline_fg_glyph.uniform_sampler(
+                        0,
+                        ImageView::new_default(tex)?,
+                        Filter::Linear,
+                    )?;
+                    let set1 = canvas.pipeline_fg_glyph.uniform_buffer(
+                        1,
+                        vec![self.fg_color.x, self.fg_color.y, self.fg_color.z, 1.],
+                    )?;
+                    let pass = canvas.pipeline_fg_glyph.create_pass(
+                        [canvas.width as _, canvas.height as _],
+                        vertex_buffer,
+                        canvas.graphics.quad_indices.clone(),
+                        vec![set0, set1],
+                    )?;
+                    cmd_buffer.run_ref(&pass)?;
+                }
+                cur_x += glyph.advance;
+            }
+            cur_y += (self.size as f32) * 1.5;
+        }
+        Ok(())
+    }
+}

src/gui/modular/button.rs

@@ -0,0 +1,618 @@
+use std::{
+    f32::consts::PI,
+    ops::Add,
+    process::{self, Child},
+    sync::Arc,
+    time::{Duration, Instant},
+};
+
+use glam::{Quat, Vec3A};
+use serde::Deserialize;
+
+use crate::{
+    backend::{
+        common::{ColorChannel, OverlaySelector, SystemTask, TaskType},
+        input::PointerMode,
+        notifications::save_notifications,
+        overlay::RelativeTo,
+    },
+    overlays::{
+        toast::Toast,
+        watch::{save_watch, WATCH_NAME},
+    },
+    state::AppState,
+};
+
+use super::{ExecArgs, ModularControl, ModularData};
+
+#[derive(Deserialize, Clone, Copy)]
+pub enum ViewAngleKind {
+    /// The cosine of the angle at which the watch becomes fully transparent
+    MinOpacity,
+    /// The cosine of the angle at which the watch becomes fully opaque
+    MaxOpacity,
+}
+
+#[derive(Deserialize, Clone, Copy)]
+pub enum Axis {
+    X,
+    Y,
+    Z,
+}
+
+#[derive(Deserialize, Clone)]
+pub enum SystemAction {
+    ToggleNotificationSounds,
+    ToggleNotifications,
+    PlayspaceResetOffset,
+    PlayspaceFixFloor,
+    RecalculateExtent,
+    PersistConfig,
+}
+
+#[derive(Deserialize, Clone)]
+pub enum WatchAction {
+    /// Hide the watch until Show/Hide binding is used
+    Hide,
+    /// Switch the watch to the opposite controller
+    SwitchHands,
+    /// Change the fade behavior of the watch
+    ViewAngle {
+        kind: ViewAngleKind,
+        delta: f32,
+    },
+    Rotation {
+        axis: Axis,
+        delta: f32,
+    },
+    Position {
+        axis: Axis,
+        delta: f32,
+    },
+}
+
+#[derive(Deserialize, Clone)]
+pub enum OverlayAction {
+    /// Reset the overlay to be in front of the HMD with its original scale
+    Reset,
+    /// Toggle the visibility of the overlay
+    ToggleVisible,
+    /// Toggle the ability to grab and recenter the overlay
+    ToggleImmovable,
+    /// Toggle the ability of the overlay to reacto to laser pointer
+    ToggleInteraction,
+    /// Change the opacity of the overlay
+    Opacity { delta: f32 },
+}
+
+#[derive(Deserialize, Clone)]
+pub enum WindowAction {
+    /// Create a new mirror window, or show/hide an existing one
+    ShowMirror,
+    /// Create a new UI window, or show/hide an existing one
+    ShowUi,
+    /// Destroy a previously created window, if it exists
+    Destroy,
+}
+
+#[derive(Deserialize, Clone)]
+#[serde(tag = "type")]
+pub enum ButtonAction {
+    Exec {
+        command: ExecArgs,
+        toast: Option<Arc<str>>,
+    },
+    Watch {
+        action: WatchAction,
+    },
+    Overlay {
+        target: OverlaySelector,
+        action: OverlayAction,
+    },
+    Window {
+        target: Arc<str>,
+        action: WindowAction,
+    },
+    Toast {
+        message: Arc<str>,
+        body: Option<Arc<str>>,
+        seconds: Option<f32>,
+    },
+    ColorAdjust {
+        channel: ColorChannel,
+        delta: f32,
+    },
+    System {
+        action: SystemAction,
+    },
+}
+
+pub(super) struct PressData {
+    last_down: Instant,
+    last_mode: PointerMode,
+    child: Option<Child>,
+}
+impl Clone for PressData {
+    fn clone(&self) -> Self {
+        Self {
+            last_down: self.last_down,
+            last_mode: self.last_mode,
+            child: None,
+        }
+    }
+}
+impl Default for PressData {
+    fn default() -> Self {
+        Self {
+            last_down: Instant::now(),
+            last_mode: PointerMode::Left,
+            child: None,
+        }
+    }
+}
+
+#[derive(Deserialize, Default, Clone)]
+pub struct ButtonData {
+    #[serde(skip)]
+    pub(super) press: PressData,
+
+    pub(super) click_down: Option<Vec<ButtonAction>>,
+    pub(super) click_up: Option<Vec<ButtonAction>>,
+    pub(super) long_click_up: Option<Vec<ButtonAction>>,
+    pub(super) right_down: Option<Vec<ButtonAction>>,
+    pub(super) right_up: Option<Vec<ButtonAction>>,
+    pub(super) long_right_up: Option<Vec<ButtonAction>>,
+    pub(super) middle_down: Option<Vec<ButtonAction>>,
+    pub(super) middle_up: Option<Vec<ButtonAction>>,
+    pub(super) long_middle_up: Option<Vec<ButtonAction>>,
+    pub(super) scroll_down: Option<Vec<ButtonAction>>,
+    pub(super) scroll_up: Option<Vec<ButtonAction>>,
+}
+
+pub fn modular_button_init(button: &mut ModularControl, data: &ButtonData) {
+    button.state = Some(ModularData::Button(Box::new(data.clone())));
+    button.on_press = Some(modular_button_dn);
+    button.on_release = Some(modular_button_up);
+    button.on_scroll = Some(modular_button_scroll);
+}
+
+fn modular_button_dn(
+    button: &mut ModularControl,
+    _: &mut (),
+    app: &mut AppState,
+    mode: PointerMode,
+) {
+    // want panic
+    let ModularData::Button(data) = button.state.as_mut().unwrap() else {
+        panic!("modular_button_dn: button state is not Button");
+    };
+
+    data.press.last_down = Instant::now();
+    data.press.last_mode = mode;
+
+    let actions = match mode {
+        PointerMode::Left => data.click_down.as_ref(),
+        PointerMode::Right => data.right_down.as_ref(),
+        PointerMode::Middle => data.middle_down.as_ref(),
+        _ => None,
+    };
+
+    if let Some(actions) = actions {
+        for action in actions {
+            handle_action(action, &mut data.press, app);
+        }
+    }
+}
+
+fn modular_button_up(button: &mut ModularControl, _: &mut (), app: &mut AppState) {
+    // want panic
+    let ModularData::Button(data) = button.state.as_mut().unwrap() else {
+        panic!("modular_button_up: button state is not Button");
+    };
+
+    let now = Instant::now();
+    let duration = now - data.press.last_down;
+    let long_press = duration.as_secs_f32() > app.session.config.long_press_duration;
+
+    let actions = match data.press.last_mode {
+        PointerMode::Left => {
+            if long_press {
+                data.long_click_up.as_ref()
+            } else {
+                data.click_up.as_ref()
+            }
+        }
+        PointerMode::Right => {
+            if long_press {
+                data.long_right_up.as_ref()
+            } else {
+                data.right_up.as_ref()
+            }
+        }
+        PointerMode::Middle => {
+            if long_press {
+                data.long_middle_up.as_ref()
+            } else {
+                data.middle_up.as_ref()
+            }
+        }
+        _ => None,
+    };
+
+    if let Some(actions) = actions {
+        for action in actions {
+            handle_action(action, &mut data.press, app);
+        }
+    }
+}
+
+fn modular_button_scroll(button: &mut ModularControl, _: &mut (), app: &mut AppState, delta: f32) {
+    // want panic
+    let ModularData::Button(data) = button.state.as_mut().unwrap() else {
+        panic!("modular_button_scroll: button state is not Button");
+    };
+
+    let actions = if delta < 0.0 {
+        data.scroll_down.as_ref()
+    } else {
+        data.scroll_up.as_ref()
+    };
+
+    if let Some(actions) = actions {
+        for action in actions {
+            handle_action(action, &mut data.press, app);
+        }
+    }
+}
+
+fn handle_action(action: &ButtonAction, press: &mut PressData, app: &mut AppState) {
+    match action {
+        ButtonAction::Exec { command, toast } => run_exec(command, toast, press, app),
+        ButtonAction::Watch { action } => run_watch(action, app),
+        ButtonAction::Overlay { target, action } => run_overlay(target, action, app),
+        ButtonAction::Window { target, action } => run_window(target, action, app),
+        ButtonAction::Toast {
+            message,
+            body,
+            seconds,
+        } => {
+            Toast::new(message.clone(), body.clone().unwrap_or_else(|| "".into()))
+                .with_timeout(seconds.unwrap_or(5.))
+                .submit(app);
+        }
+        ButtonAction::ColorAdjust { channel, delta } => {
+            let channel = *channel;
+            let delta = *delta;
+            app.tasks
+                .enqueue(TaskType::System(SystemTask::ColorGain(channel, delta)));
+        }
+        ButtonAction::System { action } => run_system(action, app),
+    }
+}
+
+fn run_system(action: &SystemAction, app: &mut AppState) {
+    match action {
+        SystemAction::PlayspaceResetOffset => {
+            app.tasks
+                .enqueue(TaskType::System(SystemTask::ResetPlayspace));
+        }
+        SystemAction::PlayspaceFixFloor => {
+            let now = Instant::now();
+            let sec = Duration::from_secs(1);
+            for i in 0..5 {
+                let at = now.add(i * sec);
+                let display = 5 - i;
+                Toast::new(
+                    format!("Fixing floor in {}", display).into(),
+                    "Place either controller on the floor.".into(),
+                )
+                .with_timeout(1.0)
+                .submit_at(app, at);
+            }
+            app.tasks
+                .enqueue_at(TaskType::System(SystemTask::FixFloor), now.add(5 * sec));
+        }
+        SystemAction::RecalculateExtent => {
+            todo!()
+        }
+        SystemAction::ToggleNotifications => {
+            app.session.config.notifications_enabled = !app.session.config.notifications_enabled;
+            Toast::new(
+                format!(
+                    "Notifications are {}.",
+                    if app.session.config.notifications_enabled {
+                        "enabled"
+                    } else {
+                        "disabled"
+                    }
+                )
+                .into(),
+                "".into(),
+            )
+            .submit(app);
+        }
+        SystemAction::ToggleNotificationSounds => {
+            app.session.config.notifications_sound_enabled =
+                !app.session.config.notifications_sound_enabled;
+            Toast::new(
+                format!(
+                    "Notification sounds are {}.",
+                    if app.session.config.notifications_sound_enabled {
+                        "enabled"
+                    } else {
+                        "disabled"
+                    }
+                )
+                .into(),
+                "".into(),
+            )
+            .submit(app);
+        }
+        SystemAction::PersistConfig => {
+            if let Err(e) = save_watch(app) {
+                log::error!("Failed to save watch config: {:?}", e);
+            };
+            if let Err(e) = save_notifications(app) {
+                log::error!("Failed to save notifications config: {:?}", e);
+            }
+        }
+    }
+}
+
+fn run_exec(args: &ExecArgs, toast: &Option<Arc<str>>, press: &mut PressData, app: &mut AppState) {
+    if let Some(proc) = press.child.as_mut() {
+        match proc.try_wait() {
+            Ok(Some(code)) => {
+                if !code.success() {
+                    log::error!("Child process exited with code: {}", code);
+                }
+                press.child = None;
+            }
+            Ok(None) => {
+                log::warn!("Unable to launch child process: previous child not exited yet");
+                return;
+            }
+            Err(e) => {
+                press.child = None;
+                log::error!("Error checking child process: {:?}", e);
+            }
+        }
+    }
+    let args = args.iter().map(|s| s.as_ref()).collect::<Vec<&str>>();
+    match process::Command::new(args[0]).args(&args[1..]).spawn() {
+        Ok(proc) => {
+            press.child = Some(proc);
+            if let Some(toast) = toast.as_ref() {
+                Toast::new(toast.clone(), "".into()).submit(app);
+            }
+        }
+        Err(e) => {
+            log::error!("Failed to spawn process {:?}: {:?}", args, e);
+        }
+    };
+}
+
+fn run_watch(data: &WatchAction, app: &mut AppState) {
+    match data {
+        WatchAction::Hide => {
+            app.tasks.enqueue(TaskType::Overlay(
+                OverlaySelector::Name(WATCH_NAME.into()),
+                Box::new(|app, o| {
+                    if o.saved_scale.is_none() {
+                        o.saved_scale = Some(o.spawn_scale);
+                        o.want_visible = false;
+                        o.saved_scale = Some(0.0);
+                        Toast::new(
+                            "Watch hidden".into(),
+                            "Use show/hide binding to restore.".into(),
+                        )
+                        .with_timeout(3.)
+                        .submit(app);
+                    } else {
+                        o.want_visible = true;
+                        o.saved_scale = None;
+                        Toast::new("Watch restored".into(), "".into()).submit(app);
+                    }
+                }),
+            ));
+            audio_thump(app);
+        }
+        WatchAction::SwitchHands => {
+            app.tasks.enqueue(TaskType::Overlay(
+                OverlaySelector::Name(WATCH_NAME.into()),
+                Box::new(|app, o| {
+                    if let RelativeTo::Hand(0) = o.relative_to {
+                        o.relative_to = RelativeTo::Hand(1);
+                        o.spawn_rotation = Quat::from_slice(&app.session.config.watch_rot)
+                            * Quat::from_rotation_x(PI)
+                            * Quat::from_rotation_z(PI);
+                        o.spawn_point = Vec3A::from_slice(&app.session.config.watch_pos);
+                        o.spawn_point.x *= -1.;
+                    } else {
+                        o.relative_to = RelativeTo::Hand(0);
+                        o.spawn_rotation = Quat::from_slice(&app.session.config.watch_rot);
+                        o.spawn_point = Vec3A::from_slice(&app.session.config.watch_pos);
+                    }
+                    o.dirty = true;
+                    Toast::new("Watch switched".into(), "Check your other hand".into())
+                        .with_timeout(3.)
+                        .submit(app);
+                }),
+            ));
+            audio_thump(app);
+        }
+        WatchAction::ViewAngle { kind, delta } => match kind {
+            ViewAngleKind::MinOpacity => {
+                let diff = (app.session.config.watch_view_angle_max
+                    - app.session.config.watch_view_angle_min)
+                    + delta;
+
+                app.session.config.watch_view_angle_min = (app.session.config.watch_view_angle_max
+                    - diff)
+                    .clamp(0.0, app.session.config.watch_view_angle_max - 0.05);
+            }
+            ViewAngleKind::MaxOpacity => {
+                let diff = app.session.config.watch_view_angle_max
+                    - app.session.config.watch_view_angle_min;
+
+                app.session.config.watch_view_angle_max =
+                    (app.session.config.watch_view_angle_max + delta).clamp(0.05, 1.0);
+
+                app.session.config.watch_view_angle_min = (app.session.config.watch_view_angle_max
+                    - diff)
+                    .clamp(0.0, app.session.config.watch_view_angle_max - 0.05);
+            }
+        },
+        WatchAction::Rotation { axis, delta } => {
+            let rot = match axis {
+                Axis::X => Quat::from_rotation_x(delta.to_radians()),
+                Axis::Y => Quat::from_rotation_y(delta.to_radians()),
+                Axis::Z => Quat::from_rotation_z(delta.to_radians()),
+            };
+            app.tasks.enqueue(TaskType::Overlay(
+                OverlaySelector::Name(WATCH_NAME.into()),
+                Box::new(move |app, o| {
+                    o.spawn_rotation *= rot;
+                    app.session.config.watch_rot = o.spawn_rotation.into();
+                    o.dirty = true;
+                }),
+            ));
+        }
+        WatchAction::Position { axis, delta } => {
+            let delta = *delta;
+            let axis = match axis {
+                Axis::X => 0,
+                Axis::Y => 1,
+                Axis::Z => 2,
+            };
+            app.tasks.enqueue(TaskType::Overlay(
+                OverlaySelector::Name(WATCH_NAME.into()),
+                Box::new(move |app, o| {
+                    o.spawn_point[axis] += delta;
+                    app.session.config.watch_pos = o.spawn_point.into();
+                    o.dirty = true;
+                }),
+            ));
+        }
+    }
+}
+
+fn run_overlay(overlay: &OverlaySelector, action: &OverlayAction, app: &mut AppState) {
+    match action {
+        OverlayAction::Reset => {
+            app.tasks.enqueue(TaskType::Overlay(
+                overlay.clone(),
+                Box::new(|app, o| {
+                    o.reset(app, true);
+                    Toast::new(format!("{} has been reset!", o.name).into(), "".into()).submit(app);
+                }),
+            ));
+        }
+        OverlayAction::ToggleVisible => {
+            app.tasks.enqueue(TaskType::Overlay(
+                overlay.clone(),
+                Box::new(|app, o| {
+                    o.want_visible = !o.want_visible;
+                    if o.recenter {
+                        o.show_hide = o.want_visible;
+                        o.reset(app, false);
+                    }
+                }),
+            ));
+        }
+        OverlayAction::ToggleImmovable => {
+            app.tasks.enqueue(TaskType::Overlay(
+                overlay.clone(),
+                Box::new(|app, o| {
+                    o.recenter = !o.recenter;
+                    o.grabbable = o.recenter;
+                    o.show_hide = o.recenter;
+                    if !o.recenter {
+                        Toast::new(
+                            format!("{} is now locked in place!", o.name).into(),
+                            "".into(),
+                        )
+                        .submit(app);
+                    } else {
+                        Toast::new(format!("{} is now unlocked!", o.name).into(), "".into())
+                            .submit(app);
+                    }
+                }),
+            ));
+            audio_thump(app);
+        }
+        OverlayAction::ToggleInteraction => {
+            app.tasks.enqueue(TaskType::Overlay(
+                overlay.clone(),
+                Box::new(|app, o| {
+                    o.interactable = !o.interactable;
+                    if !o.interactable {
+                        Toast::new(
+                            format!("{} is now non-interactable!", o.name).into(),
+                            "".into(),
+                        )
+                        .submit(app);
+                    } else {
+                        Toast::new(format!("{} is now interactable!", o.name).into(), "".into())
+                            .submit(app);
+                    }
+                }),
+            ));
+            audio_thump(app);
+        }
+        OverlayAction::Opacity { delta } => {
+            let delta = *delta;
+            app.tasks.enqueue(TaskType::Overlay(
+                overlay.clone(),
+                Box::new(move |_, o| {
+                    o.alpha = (o.alpha + delta).clamp(0.1, 1.0);
+                    o.dirty = true;
+                    log::debug!("{}: alpha {}", o.name, o.alpha);
+                }),
+            ));
+        }
+    }
+}
+
+fn run_window(window: &Arc<str>, action: &WindowAction, app: &mut AppState) {
+    use crate::overlays::custom;
+
+    match action {
+        WindowAction::ShowMirror => {
+            #[cfg(feature = "wayland")]
+            app.tasks.enqueue(TaskType::CreateOverlay(
+                OverlaySelector::Name(window.clone()),
+                Box::new({
+                    let name = window.clone();
+                    move |app| {
+                        Toast::new("Check your desktop for popup.".into(), "".into())
+                            .with_sound(true)
+                            .submit(app);
+                        crate::overlays::mirror::new_mirror(name.clone(), false, &app.session)
+                    }
+                }),
+            ));
+            #[cfg(not(feature = "wayland"))]
+            log::warn!("Mirror not available without Wayland feature.");
+        }
+        WindowAction::ShowUi => {
+            app.tasks.enqueue(TaskType::CreateOverlay(
+                OverlaySelector::Name(window.clone()),
+                Box::new({
+                    let name = window.clone();
+                    move |app| custom::create_custom(app, name)
+                }),
+            ));
+        }
+        WindowAction::Destroy => {
+            app.tasks
+                .enqueue(TaskType::DropOverlay(OverlaySelector::Name(window.clone())));
+        }
+    }
+}
+
+const THUMP_AUDIO_WAV: &[u8] = include_bytes!("../../res/380885.wav");
+
+fn audio_thump(app: &mut AppState) {
+    app.audio.play(THUMP_AUDIO_WAV);
+}

src/gui/modular/label.rs

@@ -0,0 +1,242 @@
+use chrono::Local;
+use chrono_tz::Tz;
+use glam::Vec3;
+use smallvec::SmallVec;
+use std::{
+    io::Read,
+    process::{self, Stdio},
+    sync::Arc,
+    time::Instant,
+};
+
+use crate::{gui::modular::FALLBACK_COLOR, state::AppState};
+
+use serde::Deserialize;
+
+use super::{color_parse_or_default, ExecArgs, ModularControl, ModularData};
+
+#[derive(Deserialize)]
+#[serde(tag = "source")]
+pub enum LabelContent {
+    Static {
+        text: Arc<str>,
+    },
+    Exec {
+        command: ExecArgs,
+        interval: f32,
+    },
+    Clock {
+        format: Arc<str>,
+        timezone: Option<Arc<str>>,
+    },
+    Timer {
+        format: Arc<str>,
+    },
+    Battery {
+        device: usize,
+        low_threshold: f32,
+        low_color: Arc<str>,
+        charging_color: Arc<str>,
+    },
+}
+
+pub enum LabelData {
+    Battery {
+        device: usize,
+        low_threshold: f32,
+        normal_color: Vec3,
+        low_color: Vec3,
+        charging_color: Vec3,
+    },
+    Clock {
+        format: Arc<str>,
+        timezone: Option<Tz>,
+    },
+    Timer {
+        format: Arc<str>,
+        start: Instant,
+    },
+    Exec {
+        last_exec: Instant,
+        interval: f32,
+        command: Vec<Arc<str>>,
+        child: Option<process::Child>,
+    },
+}
+
+pub fn modular_label_init(label: &mut ModularControl, content: &LabelContent) {
+    let state = match content {
+        LabelContent::Battery {
+            device,
+            low_threshold,
+            low_color,
+            charging_color,
+        } => Some(LabelData::Battery {
+            device: *device,
+            low_threshold: *low_threshold,
+            normal_color: label.fg_color,
+            low_color: color_parse_or_default(low_color),
+            charging_color: color_parse_or_default(charging_color),
+        }),
+        LabelContent::Clock { format, timezone } => {
+            let tz: Option<Tz> = timezone.as_ref().map(|tz| {
+                tz.parse().unwrap_or_else(|_| {
+                    log::error!("Failed to parse timezone '{}'", &tz);
+                    label.set_fg_color(FALLBACK_COLOR);
+                    Tz::UTC
+                })
+            });
+
+            Some(LabelData::Clock {
+                format: format.clone(),
+                timezone: tz,
+            })
+        }
+        LabelContent::Timer { format } => Some(LabelData::Timer {
+            format: format.clone(),
+            start: Instant::now(),
+        }),
+        LabelContent::Exec { command, interval } => Some(LabelData::Exec {
+            last_exec: Instant::now(),
+            interval: *interval,
+            command: command.clone(),
+            child: None,
+        }),
+        LabelContent::Static { text } => {
+            label.set_text(text);
+            None
+        }
+    };
+
+    if let Some(state) = state {
+        label.state = Some(ModularData::Label(Box::new(state)));
+        label.on_update = Some(label_update);
+    }
+}
+
+pub(super) fn label_update(control: &mut ModularControl, _: &mut (), app: &mut AppState) {
+    let ModularData::Label(data) = control.state.as_mut().unwrap() else {
+        panic!("Label control has no state");
+    };
+    match data.as_mut() {
+        LabelData::Battery {
+            device,
+            low_threshold,
+            normal_color,
+            low_color,
+            charging_color,
+        } => {
+            let device = app.input_state.devices.get(*device);
+
+            let tags = ["", "H", "L", "R", "T"];
+
+            if let Some(device) = device {
+                let (text, color) = device
+                    .soc
+                    .map(|soc| {
+                        let text = format!(
+                            "{}{}",
+                            tags[device.role as usize],
+                            (soc * 100.).min(99.) as u32
+                        );
+                        let color = if device.charging {
+                            *charging_color
+                        } else if soc < *low_threshold {
+                            *low_color
+                        } else {
+                            *normal_color
+                        };
+                        (text, color)
+                    })
+                    .unwrap_or_else(|| ("".into(), Vec3::ZERO));
+
+                control.set_text(&text);
+                control.set_fg_color(color);
+            } else {
+                control.set_text("");
+            }
+        }
+        LabelData::Clock { format, timezone } => {
+            let format = format.clone();
+            if let Some(tz) = timezone {
+                let date = Local::now().with_timezone(tz);
+                control.set_text(&format!("{}", &date.format(&format)));
+            } else {
+                let date = Local::now();
+                control.set_text(&format!("{}", &date.format(&format)));
+            }
+        }
+        LabelData::Timer { format, start } => {
+            let mut format = format.clone().to_lowercase();
+            let duration = start.elapsed().as_secs();
+            format = format.replace("%s", &format!("{:02}", (duration % 60)));
+            format = format.replace("%m", &format!("{:02}", ((duration / 60) % 60)));
+            format = format.replace("%h", &format!("{:02}", ((duration / 60) / 60)));
+            control.set_text(&format);
+        }
+        LabelData::Exec {
+            last_exec,
+            interval,
+            command,
+            child,
+        } => {
+            if let Some(mut proc) = child.take() {
+                match proc.try_wait() {
+                    Ok(Some(code)) => {
+                        if !code.success() {
+                            log::error!("Child process exited with code: {}", code);
+                        } else {
+                            if let Some(mut stdout) = proc.stdout.take() {
+                                let mut buf = String::new();
+                                if stdout.read_to_string(&mut buf).is_ok() {
+                                    control.set_text(&buf);
+                                } else {
+                                    log::error!("Failed to read stdout for child process");
+                                    return;
+                                }
+                                return;
+                            }
+                            log::error!("No stdout for child process");
+                            return;
+                        }
+                    }
+                    Ok(None) => {
+                        *child = Some(proc);
+                        // not exited yet
+                        return;
+                    }
+                    Err(e) => {
+                        *child = None;
+                        log::error!("Error checking child process: {:?}", e);
+                        return;
+                    }
+                }
+            }
+
+            if Instant::now()
+                .saturating_duration_since(*last_exec)
+                .as_secs_f32()
+                > *interval
+            {
+                *last_exec = Instant::now();
+                let args = command
+                    .iter()
+                    .map(|s| s.as_ref())
+                    .collect::<SmallVec<[&str; 8]>>();
+
+                match process::Command::new(args[0])
+                    .args(&args[1..])
+                    .stdout(Stdio::piped())
+                    .spawn()
+                {
+                    Ok(proc) => {
+                        *child = Some(proc);
+                    }
+                    Err(e) => {
+                        log::error!("Failed to spawn process {:?}: {:?}", args, e);
+                    }
+                };
+            }
+        }
+    }
+}

src/gui/modular/mod.rs

@@ -0,0 +1,334 @@
+pub mod button;
+pub mod label;
+//pub mod slider;
+
+use std::sync::Arc;
+
+use glam::Vec3;
+use serde::Deserialize;
+
+use crate::{backend::common::OverlaySelector, state::AppState};
+
+use self::{
+    button::{modular_button_init, ButtonAction, ButtonData, OverlayAction},
+    label::{modular_label_init, LabelContent, LabelData},
+};
+
+use super::{color_parse, Canvas, CanvasBuilder, Control};
+
+type ModularControl = Control<(), ModularData>;
+type ExecArgs = Vec<Arc<str>>;
+
+const FALLBACK_COLOR: Vec3 = Vec3 {
+    x: 1.,
+    y: 0.,
+    z: 1.,
+};
+
+#[derive(Deserialize)]
+pub struct ModularUiConfig {
+    pub width: f32,
+    pub size: [u32; 2],
+    pub spawn_pos: Option<[f32; 3]>,
+    pub elements: Vec<ModularElement>,
+}
+
+#[derive(Deserialize)]
+pub struct OverlayListTemplate {
+    click_down: Option<OverlayAction>,
+    click_up: Option<OverlayAction>,
+    long_click_up: Option<OverlayAction>,
+    right_down: Option<OverlayAction>,
+    right_up: Option<OverlayAction>,
+    long_right_up: Option<OverlayAction>,
+    middle_down: Option<OverlayAction>,
+    middle_up: Option<OverlayAction>,
+    long_middle_up: Option<OverlayAction>,
+    scroll_down: Option<OverlayAction>,
+    scroll_up: Option<OverlayAction>,
+}
+
+#[allow(dead_code)]
+#[derive(Deserialize)]
+#[serde(tag = "type")]
+pub enum ModularElement {
+    Panel {
+        rect: [f32; 4],
+        bg_color: Arc<str>,
+    },
+    Label {
+        rect: [f32; 4],
+        font_size: isize,
+        fg_color: Arc<str>,
+        #[serde(flatten)]
+        data: LabelContent,
+    },
+    Button {
+        rect: [f32; 4],
+        font_size: isize,
+        fg_color: Arc<str>,
+        bg_color: Arc<str>,
+        text: Arc<str>,
+        #[serde(flatten)]
+        data: Box<ButtonData>,
+    },
+    /// Convenience type to save you from having to create a bunch of labels
+    BatteryList {
+        rect: [f32; 4],
+        font_size: isize,
+        fg_color: Arc<str>,
+        fg_color_low: Arc<str>,
+        fg_color_charging: Arc<str>,
+        low_threshold: f32,
+        num_devices: usize,
+        layout: ListLayout,
+    },
+    OverlayList {
+        rect: [f32; 4],
+        font_size: isize,
+        fg_color: Arc<str>,
+        bg_color: Arc<str>,
+        layout: ListLayout,
+        #[serde(flatten)]
+        template: Box<OverlayListTemplate>,
+    },
+}
+
+#[derive(Deserialize, Clone)]
+pub enum ButtonFunc {
+    HideWatch,
+    SwitchWatchHand,
+}
+
+#[derive(Deserialize)]
+pub enum ListLayout {
+    Horizontal,
+    Vertical,
+}
+
+pub enum ModularData {
+    Label(Box<LabelData>),
+    Button(Box<ButtonData>),
+}
+
+pub fn modular_canvas(
+    size: &[u32; 2],
+    elements: &[ModularElement],
+    state: &AppState,
+) -> anyhow::Result<Canvas<(), ModularData>> {
+    let mut canvas = CanvasBuilder::new(
+        size[0] as _,
+        size[1] as _,
+        state.graphics.clone(),
+        state.format,
+        (),
+    )?;
+    let empty_str: Arc<str> = Arc::from("");
+    for elem in elements.iter() {
+        match elem {
+            ModularElement::Panel {
+                rect: [x, y, w, h],
+                bg_color,
+            } => {
+                canvas.bg_color = color_parse(bg_color).unwrap_or(FALLBACK_COLOR);
+                canvas.panel(*x, *y, *w, *h);
+            }
+            ModularElement::Label {
+                rect: [x, y, w, h],
+                font_size,
+                fg_color,
+                data,
+            } => {
+                canvas.font_size = *font_size;
+                canvas.fg_color = color_parse(fg_color).unwrap_or(FALLBACK_COLOR);
+                let label = canvas.label(*x, *y, *w, *h, empty_str.clone());
+                modular_label_init(label, data);
+            }
+            ModularElement::Button {
+                rect: [x, y, w, h],
+                font_size,
+                bg_color,
+                fg_color,
+                text,
+                data,
+            } => {
+                canvas.bg_color = color_parse(bg_color).unwrap_or(FALLBACK_COLOR);
+                canvas.fg_color = color_parse(fg_color).unwrap_or(FALLBACK_COLOR);
+                canvas.font_size = *font_size;
+                let button = canvas.button(*x, *y, *w, *h, text.clone());
+                modular_button_init(button, data);
+            }
+            ModularElement::BatteryList {
+                rect: [x, y, w, h],
+                font_size,
+                fg_color,
+                fg_color_low,
+                fg_color_charging,
+                low_threshold,
+                num_devices,
+                layout,
+            } => {
+                let num_buttons = *num_devices as f32;
+                let mut button_x = *x;
+                let mut button_y = *y;
+                let low_threshold = low_threshold * 0.01;
+                let (button_w, button_h) = match layout {
+                    ListLayout::Horizontal => (*w / num_buttons, *h),
+                    ListLayout::Vertical => (*w, *h / num_buttons),
+                };
+
+                let fg_color = color_parse(fg_color).unwrap_or(FALLBACK_COLOR);
+                canvas.font_size = *font_size;
+                canvas.fg_color = fg_color;
+
+                for i in 0..*num_devices {
+                    let label = canvas.label_centered(
+                        button_x + 2.,
+                        button_y + 2.,
+                        button_w - 4.,
+                        button_h - 4.,
+                        empty_str.clone(),
+                    );
+                    modular_label_init(
+                        label,
+                        &LabelContent::Battery {
+                            device: i,
+                            low_threshold,
+                            low_color: fg_color_low.clone(),
+                            charging_color: fg_color_charging.clone(),
+                        },
+                    );
+
+                    button_x += match layout {
+                        ListLayout::Horizontal => button_w,
+                        ListLayout::Vertical => 0.,
+                    };
+                    button_y += match layout {
+                        ListLayout::Horizontal => 0.,
+                        ListLayout::Vertical => button_h,
+                    };
+                }
+            }
+            ModularElement::OverlayList {
+                rect: [x, y, w, h],
+                font_size,
+                fg_color,
+                bg_color,
+                layout,
+                template,
+            } => {
+                let num_buttons = state.screens.len() as f32;
+                let mut button_x = *x;
+                let mut button_y = *y;
+                let (button_w, button_h) = match layout {
+                    ListLayout::Horizontal => (*w / num_buttons, *h),
+                    ListLayout::Vertical => (*w, *h / num_buttons),
+                };
+
+                canvas.bg_color = color_parse(bg_color).unwrap_or(FALLBACK_COLOR);
+                canvas.fg_color = color_parse(fg_color).unwrap_or(FALLBACK_COLOR);
+                canvas.font_size = *font_size;
+
+                for screen in state.screens.iter() {
+                    let button = canvas.button(
+                        button_x + 2.,
+                        button_y + 2.,
+                        button_w - 4.,
+                        button_h - 4.,
+                        screen.name.clone(),
+                    );
+
+                    // cursed
+                    let data = ButtonData {
+                        click_down: template.click_down.as_ref().map(|f| {
+                            vec![ButtonAction::Overlay {
+                                target: OverlaySelector::Id(screen.id),
+                                action: f.clone(),
+                            }]
+                        }),
+                        click_up: template.click_up.as_ref().map(|f| {
+                            vec![ButtonAction::Overlay {
+                                target: OverlaySelector::Id(screen.id),
+                                action: f.clone(),
+                            }]
+                        }),
+                        long_click_up: template.long_click_up.as_ref().map(|f| {
+                            vec![ButtonAction::Overlay {
+                                target: OverlaySelector::Id(screen.id),
+                                action: f.clone(),
+                            }]
+                        }),
+                        right_down: template.right_down.as_ref().map(|f| {
+                            vec![ButtonAction::Overlay {
+                                target: OverlaySelector::Id(screen.id),
+                                action: f.clone(),
+                            }]
+                        }),
+                        right_up: template.right_up.as_ref().map(|f| {
+                            vec![ButtonAction::Overlay {
+                                target: OverlaySelector::Id(screen.id),
+                                action: f.clone(),
+                            }]
+                        }),
+                        long_right_up: template.long_right_up.as_ref().map(|f| {
+                            vec![ButtonAction::Overlay {
+                                target: OverlaySelector::Id(screen.id),
+                                action: f.clone(),
+                            }]
+                        }),
+                        middle_down: template.middle_down.as_ref().map(|f| {
+                            vec![ButtonAction::Overlay {
+                                target: OverlaySelector::Id(screen.id),
+                                action: f.clone(),
+                            }]
+                        }),
+                        middle_up: template.middle_up.as_ref().map(|f| {
+                            vec![ButtonAction::Overlay {
+                                target: OverlaySelector::Id(screen.id),
+                                action: f.clone(),
+                            }]
+                        }),
+                        long_middle_up: template.long_middle_up.as_ref().map(|f| {
+                            vec![ButtonAction::Overlay {
+                                target: OverlaySelector::Id(screen.id),
+                                action: f.clone(),
+                            }]
+                        }),
+                        scroll_down: template.scroll_down.as_ref().map(|f| {
+                            vec![ButtonAction::Overlay {
+                                target: OverlaySelector::Id(screen.id),
+                                action: f.clone(),
+                            }]
+                        }),
+                        scroll_up: template.scroll_up.as_ref().map(|f| {
+                            vec![ButtonAction::Overlay {
+                                target: OverlaySelector::Id(screen.id),
+                                action: f.clone(),
+                            }]
+                        }),
+                        ..Default::default()
+                    };
+
+                    modular_button_init(button, &data);
+
+                    button_x += match layout {
+                        ListLayout::Horizontal => button_w,
+                        ListLayout::Vertical => 0.,
+                    };
+                    button_y += match layout {
+                        ListLayout::Horizontal => 0.,
+                        ListLayout::Vertical => button_h,
+                    };
+                }
+            }
+        }
+    }
+    Ok(canvas.build())
+}
+
+pub fn color_parse_or_default(color: &str) -> Vec3 {
+    color_parse(color).unwrap_or_else(|e| {
+        log::error!("Failed to parse color '{}': {}", color, e);
+        FALLBACK_COLOR
+    })
+}

src/hid.rs

@@ -0,0 +1,442 @@
+use glam::Vec2;
+use idmap::{idmap, IdMap};
+use idmap_derive::IntegerId;
+use input_linux::{
+    AbsoluteAxis, AbsoluteInfo, AbsoluteInfoSetup, EventKind, InputId, Key, RelativeAxis,
+    UInputHandle,
+};
+use libc::{input_event, timeval};
+use once_cell::sync::Lazy;
+use std::fs::File;
+use std::mem::transmute;
+use strum::{EnumIter, EnumString, IntoEnumIterator};
+
+pub fn initialize() -> Box<dyn HidProvider> {
+    if let Some(uinput) = UInputProvider::try_new() {
+        log::info!("Initialized uinput.");
+        return Box::new(uinput);
+    }
+    log::error!("Could not create uinput provider. Keyboard/Mouse input will not work!");
+    log::error!("To check if you're in input group, run: id -nG");
+    if let Ok(user) = std::env::var("USER") {
+        log::error!(
+            "To add yourself to the input group, run: sudo usermod -aG input {}",
+            user
+        );
+        log::error!("After adding yourself to the input group, you will need to reboot.");
+    }
+    Box::new(DummyProvider {})
+}
+
+pub trait HidProvider {
+    fn mouse_move(&mut self, pos: Vec2);
+    fn send_button(&self, button: u16, down: bool);
+    fn wheel(&self, delta: i32);
+    fn set_modifiers(&mut self, mods: u8);
+    fn send_key(&self, key: u16, down: bool);
+    fn set_desktop_extent(&mut self, extent: Vec2);
+    fn on_new_frame(&mut self);
+}
+
+pub struct UInputProvider {
+    handle: UInputHandle<File>,
+    desktop_extent: Vec2,
+    mouse_moved: bool,
+    cur_modifiers: u8,
+}
+
+pub struct DummyProvider;
+
+pub const MOUSE_LEFT: u16 = 0x110;
+pub const MOUSE_RIGHT: u16 = 0x111;
+pub const MOUSE_MIDDLE: u16 = 0x112;
+
+const MOUSE_EXTENT: f32 = 32768.;
+
+const EV_SYN: u16 = 0x0;
+const EV_KEY: u16 = 0x1;
+const EV_REL: u16 = 0x2;
+const EV_ABS: u16 = 0x3;
+
+impl UInputProvider {
+    fn try_new() -> Option<Self> {
+        if let Ok(file) = File::create("/dev/uinput") {
+            let handle = UInputHandle::new(file);
+
+            let id = InputId {
+                bustype: 0x03,
+                vendor: 0x4711,
+                product: 0x0829,
+                version: 5,
+            };
+
+            let name = b"WlxOverlay-S Keyboard-Mouse Hybrid Thing\0";
+
+            let abs_info = vec![
+                AbsoluteInfoSetup {
+                    axis: input_linux::AbsoluteAxis::X,
+                    info: AbsoluteInfo {
+                        value: 0,
+                        minimum: 0,
+                        maximum: MOUSE_EXTENT as _,
+                        fuzz: 0,
+                        flat: 0,
+                        resolution: 10,
+                    },
+                },
+                AbsoluteInfoSetup {
+                    axis: input_linux::AbsoluteAxis::Y,
+                    info: AbsoluteInfo {
+                        value: 0,
+                        minimum: 0,
+                        maximum: MOUSE_EXTENT as _,
+                        fuzz: 0,
+                        flat: 0,
+                        resolution: 10,
+                    },
+                },
+            ];
+
+            if handle.set_evbit(EventKind::Key).is_err() {
+                return None;
+            }
+            if handle.set_evbit(EventKind::Absolute).is_err() {
+                return None;
+            }
+            if handle.set_evbit(EventKind::Relative).is_err() {
+                return None;
+            }
+
+            for btn in MOUSE_LEFT..=MOUSE_MIDDLE {
+                let key: Key = unsafe { transmute(btn) };
+                if handle.set_keybit(key).is_err() {
+                    return None;
+                }
+            }
+
+            for key in VirtualKey::iter() {
+                let key: Key = unsafe { transmute((key as u16) - 8) };
+                if handle.set_keybit(key).is_err() {
+                    return None;
+                }
+            }
+
+            if handle.set_absbit(AbsoluteAxis::X).is_err() {
+                return None;
+            }
+            if handle.set_absbit(AbsoluteAxis::Y).is_err() {
+                return None;
+            }
+            if handle.set_relbit(RelativeAxis::Wheel).is_err() {
+                return None;
+            }
+
+            if handle.create(&id, name, 0, &abs_info).is_ok() {
+                return Some(UInputProvider {
+                    handle,
+                    desktop_extent: Vec2::ZERO,
+                    mouse_moved: false,
+                    cur_modifiers: 0,
+                });
+            }
+        }
+        None
+    }
+}
+
+impl HidProvider for UInputProvider {
+    fn mouse_move(&mut self, pos: Vec2) {
+        if self.mouse_moved {
+            return;
+        }
+        self.mouse_moved = true;
+
+        #[cfg(debug_assertions)]
+        log::trace!("Mouse move: {:?}", pos);
+
+        let pos = pos * (MOUSE_EXTENT / self.desktop_extent);
+
+        let time = get_time();
+        let events = [
+            new_event(time, EV_ABS, AbsoluteAxis::X as _, pos.x as i32),
+            new_event(time, EV_ABS, AbsoluteAxis::Y as _, pos.y as i32),
+            new_event(time, EV_SYN, 0, 0),
+        ];
+        if let Err(res) = self.handle.write(&events) {
+            log::error!("{}", res.to_string());
+        }
+    }
+    fn send_button(&self, button: u16, down: bool) {
+        let time = get_time();
+        let events = [
+            new_event(time, EV_KEY, button, down as _),
+            new_event(time, EV_SYN, 0, 0),
+        ];
+        if let Err(res) = self.handle.write(&events) {
+            log::error!("send_button: {}", res.to_string());
+        }
+    }
+    fn wheel(&self, delta: i32) {
+        let time = get_time();
+        let events = [
+            new_event(time, EV_REL, RelativeAxis::Wheel as _, delta),
+            new_event(time, EV_SYN, 0, 0),
+        ];
+        if let Err(res) = self.handle.write(&events) {
+            log::error!("wheel: {}", res.to_string());
+        }
+    }
+    fn set_modifiers(&mut self, modifiers: u8) {
+        let changed = self.cur_modifiers ^ modifiers;
+        for i in 0..7 {
+            let m = 1 << i;
+            if changed & m != 0 {
+                if let Some(vk) = MODS_TO_KEYS.get(m).into_iter().flatten().next() {
+                    self.send_key(*vk as u16, modifiers & m != 0);
+                }
+            }
+        }
+        self.cur_modifiers = modifiers;
+    }
+    fn send_key(&self, key: u16, down: bool) {
+        let time = get_time();
+        let events = [
+            new_event(time, EV_KEY, key - 8, down as _),
+            new_event(time, EV_SYN, 0, 0),
+        ];
+        if let Err(res) = self.handle.write(&events) {
+            log::error!("send_key: {}", res.to_string());
+        }
+    }
+    fn set_desktop_extent(&mut self, extent: Vec2) {
+        log::info!("Desktop extent: {:?}", extent);
+        self.desktop_extent = extent;
+    }
+    fn on_new_frame(&mut self) {
+        self.mouse_moved = false;
+    }
+}
+
+impl HidProvider for DummyProvider {
+    fn mouse_move(&mut self, _pos: Vec2) {}
+    fn send_button(&self, _button: u16, _down: bool) {}
+    fn wheel(&self, _delta: i32) {}
+    fn set_modifiers(&mut self, _modifiers: u8) {}
+    fn send_key(&self, _key: u16, _down: bool) {}
+    fn set_desktop_extent(&mut self, _extent: Vec2) {}
+    fn on_new_frame(&mut self) {}
+}
+
+#[inline]
+fn get_time() -> timeval {
+    let mut time = timeval {
+        tv_sec: 0,
+        tv_usec: 0,
+    };
+    unsafe { libc::gettimeofday(&mut time, std::ptr::null_mut()) };
+    time
+}
+
+#[inline]
+fn new_event(time: timeval, type_: u16, code: u16, value: i32) -> input_event {
+    input_event {
+        time,
+        type_,
+        code,
+        value,
+    }
+}
+
+pub type KeyModifier = u8;
+pub const SHIFT: KeyModifier = 0x01;
+pub const CAPS_LOCK: KeyModifier = 0x02;
+pub const CTRL: KeyModifier = 0x04;
+pub const ALT: KeyModifier = 0x08;
+pub const NUM_LOCK: KeyModifier = 0x10;
+pub const SUPER: KeyModifier = 0x40;
+pub const META: KeyModifier = 0x80;
+
+#[allow(non_camel_case_types)]
+#[repr(u16)]
+#[derive(Debug, PartialEq, Clone, Copy, IntegerId, EnumString, EnumIter)]
+pub enum VirtualKey {
+    Escape = 9,
+    N1, // number row
+    N2,
+    N3,
+    N4,
+    N5,
+    N6,
+    N7,
+    N8,
+    N9,
+    N0,
+    Minus,
+    Plus,
+    BackSpace,
+    Tab,
+    Q,
+    W,
+    E,
+    R,
+    T,
+    Y,
+    U,
+    I,
+    O,
+    P,
+    Oem4, // [ {
+    Oem6, // ] }
+    Return,
+    LCtrl,
+    A,
+    S,
+    D,
+    F,
+    G,
+    H,
+    J,
+    K,
+    L,
+    Oem1, // ; :
+    Oem7, // ' "
+    Oem3, // ` ~
+    LShift,
+    Oem5, // \ |
+    Z,
+    X,
+    C,
+    V,
+    B,
+    N,
+    M,
+    Comma,  // , <
+    Period, // . >
+    Oem2,   // / ?
+    RShift,
+    KP_Multiply,
+    LAlt,
+    Space,
+    Caps,
+    F1,
+    F2,
+    F3,
+    F4,
+    F5,
+    F6,
+    F7,
+    F8,
+    F9,
+    F10,
+    NumLock,
+    Scroll,
+    KP_7, // KeyPad
+    KP_8,
+    KP_9,
+    KP_Subtract,
+    KP_4,
+    KP_5,
+    KP_6,
+    KP_Add,
+    KP_1,
+    KP_2,
+    KP_3,
+    KP_0,
+    KP_Decimal,
+    Oem102 = 94, // Optional key usually between LShift and Z
+    F11,
+    F12,
+    AbntC1,
+    Katakana,
+    Hiragana,
+    Henkan,
+    Kana,
+    Muhenkan,
+    KP_Enter = 104,
+    RCtrl,
+    KP_Divide,
+    Print,
+    Meta, // Right Alt aka AltGr
+    Home = 110,
+    Up,
+    Prior,
+    Left,
+    Right,
+    End,
+    Down,
+    Next,
+    Insert,
+    Delete,
+    XF86AudioMute = 121,
+    XF86AudioLowerVolume,
+    XF86AudioRaiseVolume,
+    Pause = 127,
+    AbntC2 = 129,
+    Hangul,
+    Hanja,
+    LSuper = 133,
+    RSuper,
+    Menu,
+    Help = 146,
+    XF86MenuKB,
+    XF86Sleep = 150,
+    XF86Xfer = 155,
+    XF86Launch1,
+    XF86Launch2,
+    XF86WWW,
+    XF86Mail = 163,
+    XF86Favorites,
+    XF86MyComputer,
+    XF86Back,
+    XF86Forward,
+    XF86AudioNext = 171,
+    XF86AudioPlay,
+    XF86AudioPrev,
+    XF86AudioStop,
+    XF86HomePage = 180,
+    XF86Reload,
+    F13 = 191,
+    F14,
+    F15,
+    F16,
+    F17,
+    F18,
+    F19,
+    F20,
+    F21,
+    F22,
+    F23,
+    F24,
+    Hyper = 207,
+    XF86Launch3,
+    XF86Launch4,
+    XF86LaunchB,
+    XF86Search = 225,
+}
+
+pub static KEYS_TO_MODS: Lazy<IdMap<VirtualKey, KeyModifier>> = Lazy::new(|| {
+    idmap! {
+        VirtualKey::LShift => SHIFT,
+        VirtualKey::RShift => SHIFT,
+        VirtualKey::Caps => CAPS_LOCK,
+        VirtualKey::LCtrl => CTRL,
+        VirtualKey::RCtrl => CTRL,
+        VirtualKey::LAlt => ALT,
+        VirtualKey::NumLock => NUM_LOCK,
+        VirtualKey::LSuper => SUPER,
+        VirtualKey::RSuper => SUPER,
+        VirtualKey::Meta => META,
+    }
+});
+
+pub static MODS_TO_KEYS: Lazy<IdMap<KeyModifier, Vec<VirtualKey>>> = Lazy::new(|| {
+    idmap! {
+        SHIFT => vec![VirtualKey::LShift, VirtualKey::RShift],
+        CAPS_LOCK => vec![VirtualKey::Caps],
+        CTRL => vec![VirtualKey::LCtrl, VirtualKey::RCtrl],
+        ALT => vec![VirtualKey::LAlt],
+        NUM_LOCK => vec![VirtualKey::NumLock],
+        SUPER => vec![VirtualKey::LSuper, VirtualKey::RSuper],
+        META => vec![VirtualKey::Meta],
+    }
+});

src/main.rs

@@ -0,0 +1,138 @@
+#[allow(dead_code)]
+mod backend;
+mod config;
+mod config_io;
+mod graphics;
+mod gui;
+mod hid;
+mod overlays;
+mod shaders;
+mod state;
+
+use std::sync::{
+    atomic::{AtomicBool, Ordering},
+    Arc,
+};
+
+use clap::Parser;
+use flexi_logger::FileSpec;
+
+/// The lightweight desktop overlay for OpenVR and OpenXR
+#[derive(Parser, Debug)]
+#[command(version, about, long_about = None)]
+struct Args {
+    #[cfg(feature = "openvr")]
+    /// Force OpenVR backend
+    #[arg(long)]
+    openvr: bool,
+
+    #[cfg(feature = "openxr")]
+    /// Force OpenXR backend
+    #[arg(long)]
+    openxr: bool,
+
+    /// Uninstall OpenVR manifest and exit
+    #[arg(long)]
+    uninstall: bool,
+
+    /// Path to write logs to
+    #[arg(short, long, value_name = "FOLDER")]
+    log_to: Option<String>,
+}
+
+fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let args = Args::parse();
+    if let Some(ref log_to) = args.log_to {
+        flexi_logger::Logger::try_with_env_or_str("info")?
+            .log_to_file(FileSpec::default().directory(log_to))
+            .log_to_stdout()
+            .start()?;
+        println!("Logging to: {}", &log_to);
+    } else {
+        flexi_logger::Logger::try_with_env_or_str("info")?.start()?;
+    }
+
+    log::info!(
+        "Welcome to {} version {}!",
+        env!("CARGO_PKG_NAME"),
+        env!("CARGO_PKG_VERSION")
+    );
+
+    #[cfg(feature = "openvr")]
+    if args.uninstall {
+        crate::backend::openvr::openvr_uninstall();
+        return Ok(());
+    }
+
+    let running = Arc::new(AtomicBool::new(true));
+    let _ = ctrlc::set_handler({
+        let running = running.clone();
+        move || {
+            running.store(false, Ordering::Relaxed);
+        }
+    });
+
+    auto_run(running, args);
+
+    Ok(())
+}
+
+fn auto_run(running: Arc<AtomicBool>, args: Args) {
+    use backend::common::BackendError;
+
+    #[cfg(feature = "openxr")]
+    if !args_get_openvr(&args) {
+        use crate::backend::openxr::openxr_run;
+        match openxr_run(running.clone()) {
+            Ok(()) => return,
+            Err(BackendError::NotSupported) => (),
+            Err(e) => {
+                log::error!("{}", e.to_string());
+                return;
+            }
+        };
+    }
+
+    #[cfg(feature = "openvr")]
+    if !args_get_openxr(&args) {
+        use crate::backend::openvr::openvr_run;
+        match openvr_run(running.clone()) {
+            Ok(()) => return,
+            Err(BackendError::NotSupported) => (),
+            Err(e) => {
+                log::error!("{}", e.to_string());
+                return;
+            }
+        };
+    }
+
+    log::error!("No more backends to try");
+
+    #[cfg(not(any(feature = "openvr", feature = "openxr")))]
+    compile_error!("No VR support! Enable either openvr or openxr features!");
+
+    #[cfg(not(any(feature = "wayland", feature = "x11")))]
+    compile_error!("No desktop support! Enable either wayland or x11 features!");
+}
+
+#[allow(dead_code)]
+fn args_get_openvr(_args: &Args) -> bool {
+    #[cfg(feature = "openvr")]
+    let ret = _args.openvr;
+
+    #[cfg(not(feature = "openvr"))]
+    let ret = false;
+
+    ret
+}
+
+#[allow(dead_code)]
+fn args_get_openxr(_args: &Args) -> bool {
+    #[cfg(feature = "openxr")]
+    let ret = _args.openxr;
+
+    #[cfg(not(feature = "openxr"))]
+    let ret = false;
+
+    ret
+}

src/overlays/custom.rs

@@ -0,0 +1,51 @@
+use std::sync::Arc;
+
+use glam::Vec3A;
+
+use crate::{
+    backend::overlay::{ui_transform, OverlayBackend, OverlayState},
+    config::{load_custom_ui, load_known_yaml, ConfigType},
+    gui::modular::{modular_canvas, ModularUiConfig},
+    state::AppState,
+};
+
+const SETTINGS_NAME: &str = "settings";
+
+pub fn create_custom(
+    state: &AppState,
+    name: Arc<str>,
+) -> Option<(OverlayState, Box<dyn OverlayBackend>)> {
+    let config = if &*name == SETTINGS_NAME {
+        load_known_yaml::<ModularUiConfig>(ConfigType::Settings)
+    } else {
+        match load_custom_ui(&name) {
+            Ok(config) => config,
+            Err(e) => {
+                log::error!("Failed to load custom UI config for {}: {:?}", name, e);
+                return None;
+            }
+        }
+    };
+
+    let canvas = match modular_canvas(&config.size, &config.elements, state) {
+        Ok(canvas) => canvas,
+        Err(e) => {
+            log::error!("Failed to create canvas for {}: {:?}", name, e);
+            return None;
+        }
+    };
+
+    let state = OverlayState {
+        name: name.clone(),
+        want_visible: true,
+        interactable: true,
+        grabbable: true,
+        spawn_scale: config.width,
+        spawn_point: Vec3A::from_array(config.spawn_pos.unwrap_or([0., 0., -0.5])),
+        interaction_transform: ui_transform(&config.size),
+        ..Default::default()
+    };
+    let backend = Box::new(canvas);
+
+    Some((state, backend))
+}

src/overlays/keyboard.rs

@@ -0,0 +1,357 @@
+use std::{
+    collections::HashMap,
+    process::{Child, Command},
+    str::FromStr,
+};
+
+use crate::{
+    backend::{
+        input::PointerMode,
+        overlay::{OverlayData, OverlayState},
+    },
+    config::{self, ConfigType},
+    gui::{color_parse, CanvasBuilder, Control},
+    hid::{KeyModifier, VirtualKey, ALT, CTRL, KEYS_TO_MODS, META, SHIFT, SUPER},
+    state::AppState,
+};
+use glam::{vec2, vec3a, Affine2, Vec4};
+use once_cell::sync::Lazy;
+use regex::Regex;
+use serde::{Deserialize, Serialize};
+
+const PIXELS_PER_UNIT: f32 = 80.;
+const BUTTON_PADDING: f32 = 4.;
+const AUTO_RELEASE_MODS: [KeyModifier; 5] = [SHIFT, CTRL, ALT, SUPER, META];
+
+pub const KEYBOARD_NAME: &str = "kbd";
+
+pub fn create_keyboard<O>(app: &AppState) -> anyhow::Result<OverlayData<O>>
+where
+    O: Default,
+{
+    let size = vec2(
+        LAYOUT.row_size * PIXELS_PER_UNIT,
+        (LAYOUT.main_layout.len() as f32) * PIXELS_PER_UNIT,
+    );
+
+    let data = KeyboardData {
+        modifiers: 0,
+        processes: vec![],
+    };
+
+    let mut canvas = CanvasBuilder::new(
+        size.x as _,
+        size.y as _,
+        app.graphics.clone(),
+        app.format,
+        data,
+    )?;
+
+    canvas.bg_color = color_parse("#101010").unwrap(); //safe
+    canvas.panel(0., 0., size.x, size.y);
+
+    canvas.font_size = 18;
+    canvas.bg_color = color_parse("#202020").unwrap(); //safe
+
+    let unit_size = size.x / LAYOUT.row_size;
+    let h = unit_size - 2. * BUTTON_PADDING;
+
+    for row in 0..LAYOUT.key_sizes.len() {
+        let y = unit_size * (row as f32) + BUTTON_PADDING;
+        let mut sum_size = 0f32;
+
+        for col in 0..LAYOUT.key_sizes[row].len() {
+            let my_size = LAYOUT.key_sizes[row][col];
+            let x = unit_size * sum_size + BUTTON_PADDING;
+            let w = unit_size * my_size - 2. * BUTTON_PADDING;
+
+            if let Some(key) = LAYOUT.main_layout[row][col].as_ref() {
+                let mut maybe_state: Option<KeyButtonData> = None;
+                if let Ok(vk) = VirtualKey::from_str(key) {
+                    if let Some(mods) = KEYS_TO_MODS.get(vk) {
+                        maybe_state = Some(KeyButtonData::Modifier {
+                            modifier: *mods,
+                            sticky: false,
+                        });
+                    } else {
+                        maybe_state = Some(KeyButtonData::Key { vk, pressed: false });
+                    }
+                } else if let Some(macro_verbs) = LAYOUT.macros.get(key) {
+                    maybe_state = Some(KeyButtonData::Macro {
+                        verbs: key_events_for_macro(macro_verbs),
+                    });
+                } else if let Some(exec_args) = LAYOUT.exec_commands.get(key) {
+                    if exec_args.is_empty() {
+                        log::error!("Keyboard: EXEC args empty for {}", key);
+                        continue;
+                    }
+                    maybe_state = Some(KeyButtonData::Exec {
+                        program: exec_args
+                            .first()
+                            .unwrap() // safe because we checked is_empty
+                            .clone(),
+                        args: exec_args.iter().skip(1).cloned().collect(),
+                    });
+                } else {
+                    log::error!("Unknown key: {}", key);
+                }
+
+                if let Some(state) = maybe_state {
+                    let label = LAYOUT.label_for_key(key);
+                    let button = canvas.key_button(x, y, w, h, &label);
+                    button.state = Some(state);
+                    button.on_press = Some(key_press);
+                    button.on_release = Some(key_release);
+                    button.test_highlight = Some(test_highlight);
+                }
+            }
+
+            sum_size += my_size;
+        }
+    }
+
+    let canvas = canvas.build();
+
+    let interaction_transform = Affine2::from_translation(vec2(0.5, 0.5))
+        * Affine2::from_scale(vec2(1., -size.x as f32 / size.y as f32));
+
+    let width = LAYOUT.row_size * 0.05 * app.session.config.keyboard_scale;
+
+    Ok(OverlayData {
+        state: OverlayState {
+            name: KEYBOARD_NAME.into(),
+            grabbable: true,
+            recenter: true,
+            interactable: true,
+            spawn_scale: width,
+            spawn_point: vec3a(0., -0.5, -1.),
+            interaction_transform,
+            ..Default::default()
+        },
+        backend: Box::new(canvas),
+        ..Default::default()
+    })
+}
+
+fn key_press(
+    control: &mut Control<KeyboardData, KeyButtonData>,
+    data: &mut KeyboardData,
+    app: &mut AppState,
+    mode: PointerMode,
+) {
+    match control.state.as_mut() {
+        Some(KeyButtonData::Key { vk, pressed }) => {
+            data.key_click(app);
+
+            if let PointerMode::Right = mode {
+                data.modifiers |= SHIFT;
+                app.hid_provider.set_modifiers(data.modifiers);
+            }
+
+            app.hid_provider.send_key(*vk as _, true);
+            *pressed = true;
+        }
+        Some(KeyButtonData::Modifier { modifier, sticky }) => {
+            *sticky = data.modifiers & *modifier == 0;
+            data.modifiers |= *modifier;
+            data.key_click(app);
+            app.hid_provider.set_modifiers(data.modifiers);
+        }
+        Some(KeyButtonData::Macro { verbs }) => {
+            data.key_click(app);
+            for (vk, press) in verbs {
+                app.hid_provider.send_key(*vk as _, *press);
+            }
+        }
+        Some(KeyButtonData::Exec { program, args }) => {
+            // Reap previous processes
+            data.processes
+                .retain_mut(|child| !matches!(child.try_wait(), Ok(Some(_))));
+
+            data.key_click(app);
+            if let Ok(child) = Command::new(program).args(args).spawn() {
+                data.processes.push(child);
+            }
+        }
+        None => {}
+    }
+}
+
+fn key_release(
+    control: &mut Control<KeyboardData, KeyButtonData>,
+    data: &mut KeyboardData,
+    app: &mut AppState,
+) {
+    match control.state.as_mut() {
+        Some(KeyButtonData::Key { vk, pressed }) => {
+            app.hid_provider.send_key(*vk as _, false);
+            *pressed = false;
+
+            for m in AUTO_RELEASE_MODS.iter() {
+                if data.modifiers & *m != 0 {
+                    data.modifiers &= !*m;
+                    app.hid_provider.set_modifiers(data.modifiers);
+                }
+            }
+        }
+        Some(KeyButtonData::Modifier { modifier, sticky }) => {
+            if !*sticky {
+                data.modifiers &= !*modifier;
+                app.hid_provider.set_modifiers(data.modifiers);
+            }
+        }
+        _ => {}
+    }
+}
+
+static PRESS_COLOR: Vec4 = Vec4::new(1.0, 1.0, 1.0, 0.5);
+
+fn test_highlight(
+    control: &Control<KeyboardData, KeyButtonData>,
+    data: &mut KeyboardData,
+    _app: &mut AppState,
+) -> Option<Vec4> {
+    let pressed = match control.state.as_ref() {
+        Some(KeyButtonData::Key { pressed, .. }) => *pressed,
+        Some(KeyButtonData::Modifier { modifier, .. }) => data.modifiers & *modifier != 0,
+        _ => false,
+    };
+
+    if pressed {
+        Some(PRESS_COLOR)
+    } else {
+        None
+    }
+}
+
+struct KeyboardData {
+    modifiers: KeyModifier,
+    processes: Vec<Child>,
+}
+
+const KEY_AUDIO_WAV: &[u8] = include_bytes!("../res/421581.wav");
+
+impl KeyboardData {
+    fn key_click(&mut self, app: &mut AppState) {
+        if app.session.config.keyboard_sound_enabled {
+            app.audio.play(KEY_AUDIO_WAV);
+        }
+    }
+}
+
+enum KeyButtonData {
+    Key { vk: VirtualKey, pressed: bool },
+    Modifier { modifier: KeyModifier, sticky: bool },
+    Macro { verbs: Vec<(VirtualKey, bool)> },
+    Exec { program: String, args: Vec<String> },
+}
+
+static LAYOUT: Lazy<Layout> = Lazy::new(Layout::load_from_disk);
+
+static MACRO_REGEX: Lazy<Regex> =
+    Lazy::new(|| Regex::new(r"^([A-Za-z0-1_-]+)(?: +(UP|DOWN))?$").unwrap()); // want panic
+
+#[derive(Debug, Deserialize, Serialize)]
+pub struct Layout {
+    name: String,
+    row_size: f32,
+    key_sizes: Vec<Vec<f32>>,
+    main_layout: Vec<Vec<Option<String>>>,
+    exec_commands: HashMap<String, Vec<String>>,
+    macros: HashMap<String, Vec<String>>,
+    labels: HashMap<String, Vec<String>>,
+}
+
+impl Layout {
+    fn load_from_disk() -> Layout {
+        let mut layout = config::load_known_yaml::<Layout>(ConfigType::Keyboard);
+        layout.post_load();
+        layout
+    }
+
+    fn post_load(&mut self) {
+        for i in 0..self.key_sizes.len() {
+            let row = &self.key_sizes[i];
+            let width: f32 = row.iter().sum();
+            if (width - self.row_size).abs() > 0.001 {
+                panic!(
+                    "Row {} has a width of {}, but the row size is {}",
+                    i, width, self.row_size
+                );
+            }
+        }
+
+        for i in 0..self.main_layout.len() {
+            let row = &self.main_layout[i];
+            let width = row.len();
+            if width != self.key_sizes[i].len() {
+                panic!(
+                    "Row {} has {} keys, needs to have {} according to key_sizes",
+                    i,
+                    width,
+                    self.key_sizes[i].len()
+                );
+            }
+        }
+    }
+
+    fn label_for_key(&self, key: &str) -> Vec<String> {
+        if let Some(label) = self.labels.get(key) {
+            return label.clone();
+        }
+        if key.is_empty() {
+            return vec![];
+        }
+        if key.len() == 1 {
+            return vec![key.to_string().to_lowercase()];
+        }
+        let mut key = key;
+        if key.starts_with("KP_") {
+            key = &key[3..];
+        }
+        if key.contains('_') {
+            key = key.split('_').next().unwrap_or_else(|| {
+                log::error!(
+                    "keyboard.yaml: Key '{}' must not start or end with '_'!",
+                    key
+                );
+                "???"
+            });
+        }
+        vec![format!(
+            "{}{}",
+            key.chars().next().unwrap().to_uppercase(), // safe because we checked is_empty
+            &key[1..].to_lowercase()
+        )]
+    }
+}
+
+fn key_events_for_macro(macro_verbs: &Vec<String>) -> Vec<(VirtualKey, bool)> {
+    let mut key_events = vec![];
+    for verb in macro_verbs {
+        if let Some(caps) = MACRO_REGEX.captures(verb) {
+            if let Ok(virtual_key) = VirtualKey::from_str(&caps[1]) {
+                if let Some(state) = caps.get(2) {
+                    if state.as_str() == "UP" {
+                        key_events.push((virtual_key, false));
+                    } else if state.as_str() == "DOWN" {
+                        key_events.push((virtual_key, true));
+                    } else {
+                        log::error!(
+                            "Unknown key state in macro: {}, looking for UP or DOWN.",
+                            state.as_str()
+                        );
+                        return vec![];
+                    }
+                } else {
+                    key_events.push((virtual_key, true));
+                    key_events.push((virtual_key, false));
+                }
+            } else {
+                log::error!("Unknown virtual key: {}", &caps[1]);
+                return vec![];
+            }
+        }
+    }
+    key_events
+}

src/overlays/mirror.rs

@@ -0,0 +1,144 @@
+use std::{
+    sync::Arc,
+    task::{Context, Poll},
+};
+
+use futures::{Future, FutureExt};
+use glam::vec3a;
+use wlx_capture::pipewire::{pipewire_select_screen, PipewireCapture, PipewireSelectScreenResult};
+
+use crate::{
+    backend::{
+        common::{OverlaySelector, TaskType},
+        overlay::{
+            ui_transform, OverlayBackend, OverlayRenderer, OverlayState, SplitOverlayBackend,
+        },
+    },
+    state::{AppSession, AppState},
+};
+
+use super::screen::ScreenRenderer;
+type PinnedSelectorFuture = core::pin::Pin<
+    Box<dyn Future<Output = Result<PipewireSelectScreenResult, wlx_capture::pipewire::AshpdError>>>,
+>;
+
+pub struct MirrorRenderer {
+    name: Arc<str>,
+    renderer: Option<ScreenRenderer>,
+    selector: Option<PinnedSelectorFuture>,
+    last_extent: [u32; 3],
+}
+impl MirrorRenderer {
+    pub fn new(name: Arc<str>) -> Self {
+        let selector = Box::pin(pipewire_select_screen(None, false, false, false));
+        Self {
+            name,
+            renderer: None,
+            selector: Some(selector),
+            last_extent: [0; 3],
+        }
+    }
+}
+
+impl OverlayRenderer for MirrorRenderer {
+    fn init(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+    fn render(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        if let Some(mut selector) = self.selector.take() {
+            let maybe_pw_result = match selector
+                .poll_unpin(&mut Context::from_waker(futures::task::noop_waker_ref()))
+            {
+                Poll::Ready(result) => result,
+                Poll::Pending => {
+                    self.selector = Some(selector);
+                    return Ok(());
+                }
+            };
+
+            if let Ok(pw_result) = maybe_pw_result {
+                log::info!(
+                    "{}: PipeWire node selected: {}",
+                    self.name.clone(),
+                    pw_result.node_id
+                );
+                let capture = PipewireCapture::new(self.name.clone(), pw_result.node_id, 60);
+                self.renderer = Some(ScreenRenderer::new_raw(
+                    self.name.clone(),
+                    Box::new(capture),
+                ));
+                app.tasks.enqueue(TaskType::Overlay(
+                    OverlaySelector::Name(self.name.clone()),
+                    Box::new(|app, o| {
+                        o.grabbable = true;
+                        o.interactable = true;
+                        o.reset(app, false);
+                    }),
+                ));
+            } else {
+                log::warn!("Failed to create pipewire mirror");
+                self.renderer = None;
+                // drop self
+                app.tasks
+                    .enqueue(TaskType::DropOverlay(OverlaySelector::Name(
+                        self.name.clone(),
+                    )));
+            }
+        }
+
+        if let Some(renderer) = self.renderer.as_mut() {
+            renderer.render(app)?;
+            if let Some(view) = renderer.view() {
+                let extent = view.image().extent();
+                if self.last_extent != extent {
+                    self.last_extent = extent;
+                    // resized
+                    app.tasks.enqueue(TaskType::Overlay(
+                        OverlaySelector::Name(self.name.clone()),
+                        Box::new(move |_app, o| {
+                            o.interaction_transform = ui_transform(&[extent[0], extent[1]]);
+                        }),
+                    ));
+                }
+            }
+        }
+
+        Ok(())
+    }
+    fn pause(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        if let Some(renderer) = self.renderer.as_mut() {
+            renderer.pause(app)?;
+        }
+        Ok(())
+    }
+    fn resume(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        if let Some(renderer) = self.renderer.as_mut() {
+            renderer.resume(app)?;
+        }
+        Ok(())
+    }
+    fn view(&mut self) -> Option<std::sync::Arc<vulkano::image::view::ImageView>> {
+        self.renderer.as_mut().and_then(|r| r.view())
+    }
+}
+
+pub fn new_mirror(
+    name: Arc<str>,
+    show_hide: bool,
+    session: &AppSession,
+) -> Option<(OverlayState, Box<dyn OverlayBackend>)> {
+    let state = OverlayState {
+        name: name.clone(),
+        show_hide,
+        want_visible: true,
+        spawn_scale: 0.5 * session.config.desktop_view_scale,
+        spawn_point: vec3a(0., 0.5, -0.5),
+        ..Default::default()
+    };
+    let backend = Box::new(SplitOverlayBackend {
+        renderer: Box::new(MirrorRenderer::new(name)),
+        ..Default::default()
+    });
+
+    Some((state, backend))
+}

src/overlays/mod.rs

@@ -0,0 +1,7 @@
+pub mod custom;
+pub mod keyboard;
+#[cfg(feature = "wayland")]
+pub mod mirror;
+pub mod screen;
+pub mod toast;
+pub mod watch;

src/overlays/screen.rs

@@ -0,0 +1,863 @@
+use core::slice;
+use serde::{Deserialize, Serialize};
+use std::{
+    ops::Add,
+    ptr,
+    sync::Arc,
+    time::{Duration, Instant},
+};
+use vulkano::{
+    command_buffer::CommandBufferUsage,
+    image::{sampler::Filter, view::ImageView, Image},
+    pipeline::graphics::color_blend::AttachmentBlend,
+};
+use wlx_capture::{
+    frame::{
+        DrmFormat, MouseMeta, WlxFrame, DRM_FORMAT_ABGR8888, DRM_FORMAT_ARGB8888,
+        DRM_FORMAT_XBGR8888, DRM_FORMAT_XRGB8888,
+    },
+    WlxCapture,
+};
+
+#[cfg(feature = "wayland")]
+use {
+    crate::config_io,
+    glam::Vec3,
+    std::{collections::HashMap, error::Error, f32::consts::PI, ops::Deref, path::PathBuf},
+    wlx_capture::{
+        pipewire::{pipewire_select_screen, PipewireCapture},
+        wayland::{wayland_client::protocol::wl_output, WlxClient, WlxOutput},
+        wlr_dmabuf::WlrDmabufCapture,
+        wlr_screencopy::WlrScreencopyCapture,
+    },
+};
+
+#[cfg(feature = "x11")]
+use wlx_capture::xshm::{XshmCapture, XshmScreen};
+
+use glam::{vec2, vec3a, Affine2, Quat, Vec2};
+
+use crate::{
+    backend::{
+        input::{Haptics, InteractionHandler, PointerHit, PointerMode},
+        overlay::{OverlayData, OverlayRenderer, OverlayState, SplitOverlayBackend},
+    },
+    config::def_pw_tokens,
+    graphics::{fourcc_to_vk, WlxCommandBuffer, WlxPipeline, WlxPipelineLegacy},
+    hid::{MOUSE_LEFT, MOUSE_MIDDLE, MOUSE_RIGHT},
+    state::{AppSession, AppState},
+};
+
+const CURSOR_SIZE: f32 = 16. / 1440.;
+
+static DRM_FORMATS: once_cell::sync::OnceCell<Vec<DrmFormat>> = once_cell::sync::OnceCell::new();
+
+pub struct ScreenInteractionHandler {
+    next_scroll: Instant,
+    next_move: Instant,
+    mouse_transform: Affine2,
+}
+impl ScreenInteractionHandler {
+    fn new(pos: Vec2, size: Vec2, transform: Transform) -> ScreenInteractionHandler {
+        let transform = match transform {
+            Transform::_90 | Transform::Flipped90 => Affine2::from_cols(
+                vec2(0., size.y),
+                vec2(-size.x, 0.),
+                vec2(pos.x + size.x, pos.y),
+            ),
+            Transform::_180 | Transform::Flipped180 => Affine2::from_cols(
+                vec2(-size.x, 0.),
+                vec2(0., -size.y),
+                vec2(pos.x + size.x, pos.y + size.y),
+            ),
+            Transform::_270 | Transform::Flipped270 => Affine2::from_cols(
+                vec2(0., -size.y),
+                vec2(size.x, 0.),
+                vec2(pos.x, pos.y + size.y),
+            ),
+            _ => Affine2::from_cols(vec2(size.x, 0.), vec2(0., size.y), pos),
+        };
+
+        ScreenInteractionHandler {
+            next_scroll: Instant::now(),
+            next_move: Instant::now(),
+            mouse_transform: transform,
+        }
+    }
+}
+
+impl InteractionHandler for ScreenInteractionHandler {
+    fn on_hover(&mut self, app: &mut AppState, hit: &PointerHit) -> Option<Haptics> {
+        #[cfg(debug_assertions)]
+        log::trace!("Hover: {:?}", hit.uv);
+        if self.next_move < Instant::now() {
+            let pos = self.mouse_transform.transform_point2(hit.uv);
+            app.hid_provider.mouse_move(pos);
+        }
+        None
+    }
+    fn on_pointer(&mut self, app: &mut AppState, hit: &PointerHit, pressed: bool) {
+        let btn = match hit.mode {
+            PointerMode::Right => MOUSE_RIGHT,
+            PointerMode::Middle => MOUSE_MIDDLE,
+            _ => MOUSE_LEFT,
+        };
+
+        if pressed {
+            self.next_move = Instant::now()
+                + Duration::from_millis(app.session.config.click_freeze_time_ms as u64);
+        }
+
+        app.hid_provider.send_button(btn, pressed);
+
+        let pos = self.mouse_transform.transform_point2(hit.uv);
+        app.hid_provider.mouse_move(pos);
+    }
+    fn on_scroll(&mut self, app: &mut AppState, hit: &PointerHit, delta: f32) {
+        if self.next_scroll > Instant::now() {
+            return;
+        }
+        let max_millis = if matches!(hit.mode, PointerMode::Right) {
+            50.0
+        } else {
+            100.0
+        };
+
+        let millis = (1. - delta.abs()) * max_millis;
+        self.next_scroll = Instant::now().add(Duration::from_millis(millis as _));
+        app.hid_provider.wheel(if delta < 0. { -1 } else { 1 })
+    }
+    fn on_left(&mut self, _app: &mut AppState, _hand: usize) {}
+}
+
+struct ScreenPipeline {
+    view: Arc<ImageView>,
+    mouse: Option<Arc<ImageView>>,
+    pipeline: Arc<WlxPipeline<WlxPipelineLegacy>>,
+    extentf: [f32; 2],
+}
+
+impl ScreenPipeline {
+    fn new(extent: &[u32; 3], app: &mut AppState) -> anyhow::Result<ScreenPipeline> {
+        let texture =
+            app.graphics
+                .render_texture(extent[0], extent[1], app.graphics.native_format)?;
+
+        let view = ImageView::new_default(texture)?;
+
+        let Ok(shaders) = app.graphics.shared_shaders.read() else {
+            return Err(anyhow::anyhow!("Could not lock shared shaders for reading"));
+        };
+
+        let pipeline = app.graphics.create_pipeline(
+            view.clone(),
+            shaders.get("vert_common").unwrap().clone(), // want panic
+            shaders.get("frag_screen").unwrap().clone(), // want panic
+            app.graphics.native_format,
+            Some(AttachmentBlend::default()),
+        )?;
+
+        let extentf = [extent[0] as f32, extent[1] as f32];
+
+        Ok(ScreenPipeline {
+            view,
+            mouse: None,
+            pipeline,
+            extentf,
+        })
+    }
+
+    fn ensure_mouse_initialized(&mut self, uploads: &mut WlxCommandBuffer) -> anyhow::Result<()> {
+        if self.mouse.is_some() {
+            return Ok(());
+        }
+
+        #[rustfmt::skip]
+        let mouse_bytes = [
+            0x00, 0x00, 0x00, 0xff,  0x00, 0x00, 0x00, 0xff,  0x00, 0x00, 0x00, 0xff,  0x00, 0x00, 0x00, 0xff,
+            0x00, 0x00, 0x00, 0xff,  0xff, 0xff, 0xff, 0xff,  0xff, 0xff, 0xff, 0xff,  0x00, 0x00, 0x00, 0xff,
+            0x00, 0x00, 0x00, 0xff,  0xff, 0xff, 0xff, 0xff,  0xff, 0xff, 0xff, 0xff,  0x00, 0x00, 0x00, 0xff,
+            0x00, 0x00, 0x00, 0xff,  0x00, 0x00, 0x00, 0xff,  0x00, 0x00, 0x00, 0xff,  0x00, 0x00, 0x00, 0xff,
+        ];
+
+        let mouse_tex =
+            uploads.texture2d(4, 4, vulkano::format::Format::R8G8B8A8_UNORM, &mouse_bytes)?;
+        self.mouse = Some(ImageView::new_default(mouse_tex)?);
+        Ok(())
+    }
+
+    fn render(
+        &mut self,
+        image: Arc<Image>,
+        mouse: Option<&MouseMeta>,
+        app: &mut AppState,
+    ) -> anyhow::Result<()> {
+        let mut cmd = app
+            .graphics
+            .create_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+        let view = ImageView::new_default(image)?;
+        let set0 = self.pipeline.uniform_sampler(0, view, Filter::Linear)?;
+
+        let pass = self.pipeline.create_pass(
+            self.extentf,
+            app.graphics.quad_verts.clone(),
+            app.graphics.quad_indices.clone(),
+            vec![set0],
+        )?;
+
+        cmd.begin_render_pass(&self.pipeline)?;
+        cmd.run_ref(&pass)?;
+
+        if let (Some(mouse), Some(mouse_view)) = (mouse, self.mouse.clone()) {
+            let size = CURSOR_SIZE * self.extentf[1];
+            let half_size = size * 0.5;
+
+            let vertex_buffer = app.graphics.upload_verts(
+                self.extentf[0],
+                self.extentf[1],
+                mouse.x * self.extentf[0] - half_size,
+                mouse.y * self.extentf[1] - half_size,
+                size,
+                size,
+            )?;
+
+            let set0 = self
+                .pipeline
+                .uniform_sampler(0, mouse_view.clone(), Filter::Nearest)?;
+
+            let pass = self.pipeline.create_pass(
+                self.extentf,
+                vertex_buffer,
+                app.graphics.quad_indices.clone(),
+                vec![set0],
+            )?;
+
+            cmd.run_ref(&pass)?;
+        }
+
+        cmd.end_render_pass()?;
+        cmd.build_and_execute_now()
+    }
+}
+
+pub struct ScreenRenderer {
+    name: Arc<str>,
+    capture: Box<dyn WlxCapture>,
+    pipeline: Option<ScreenPipeline>,
+    last_view: Option<Arc<ImageView>>,
+    extent: [u32; 3],
+}
+
+impl ScreenRenderer {
+    #[cfg(feature = "wayland")]
+    pub fn new_raw(name: Arc<str>, capture: Box<dyn WlxCapture>) -> ScreenRenderer {
+        ScreenRenderer {
+            name,
+            capture,
+            pipeline: None,
+            last_view: None,
+            extent: [0; 3],
+        }
+    }
+
+    #[cfg(feature = "wayland")]
+    pub fn new_wlr_dmabuf(output: &WlxOutput) -> Option<ScreenRenderer> {
+        let client = WlxClient::new()?;
+        let capture = WlrDmabufCapture::new(client, output.id);
+
+        Some(ScreenRenderer {
+            name: output.name.clone(),
+            capture: Box::new(capture),
+            pipeline: None,
+            last_view: None,
+            extent: extent_from_res(output.size),
+        })
+    }
+
+    #[cfg(feature = "wayland")]
+    pub fn new_wlr_screencopy(output: &WlxOutput) -> Option<ScreenRenderer> {
+        let client = WlxClient::new()?;
+        let capture = WlrScreencopyCapture::new(client, output.id);
+
+        Some(ScreenRenderer {
+            name: output.name.clone(),
+            capture: Box::new(capture),
+            pipeline: None,
+            last_view: None,
+            extent: extent_from_res(output.size),
+        })
+    }
+
+    #[cfg(feature = "wayland")]
+    pub fn new_pw(
+        output: &WlxOutput,
+        token: Option<&str>,
+        session: &AppSession,
+    ) -> anyhow::Result<(
+        ScreenRenderer,
+        Option<String>, /* pipewire restore token */
+    )> {
+        let name = output.name.clone();
+        let embed_mouse = !session.config.double_cursor_fix;
+        let select_screen_result =
+            futures::executor::block_on(pipewire_select_screen(token, embed_mouse, true, true))?;
+
+        let capture = PipewireCapture::new(name, select_screen_result.node_id, 60);
+
+        Ok((
+            ScreenRenderer {
+                name: output.name.clone(),
+                capture: Box::new(capture),
+                pipeline: None,
+                last_view: None,
+                extent: extent_from_res(output.size),
+            },
+            select_screen_result.restore_token,
+        ))
+    }
+
+    #[cfg(feature = "x11")]
+    pub fn new_xshm(screen: Arc<XshmScreen>) -> ScreenRenderer {
+        let capture = XshmCapture::new(screen.clone());
+
+        ScreenRenderer {
+            name: screen.name.clone(),
+            capture: Box::new(capture),
+            pipeline: None,
+            last_view: None,
+            extent: extent_from_res((screen.monitor.width(), screen.monitor.height())),
+        }
+    }
+}
+
+impl OverlayRenderer for ScreenRenderer {
+    fn init(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        Ok(())
+    }
+    fn render(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        if !self.capture.is_ready() {
+            let supports_dmabuf = self.capture.supports_dmbuf();
+            let allow_dmabuf = &*app.session.config.capture_method != "pw_fallback"
+                && &*app.session.config.capture_method != "screencopy";
+
+            let drm_formats = DRM_FORMATS.get_or_init({
+                let graphics = app.graphics.clone();
+                move || {
+                    if !supports_dmabuf {
+                        log::info!("Capture method does not support DMA-buf");
+                        return vec![];
+                    }
+                    if !allow_dmabuf {
+                        log::info!("Not using DMA-buf capture due to pw_fallback");
+                        return vec![];
+                    }
+                    log::warn!("Using DMA-buf capture. If screens are blank for you, switch to SHM using:");
+                    log::warn!("echo 'capture_method: pw_fallback' > ~/.config/wlxoverlay/conf.d/pw_fallback.yaml");
+
+                    let possible_formats = [
+                        DRM_FORMAT_ABGR8888.into(),
+                        DRM_FORMAT_XBGR8888.into(),
+                        DRM_FORMAT_ARGB8888.into(),
+                        DRM_FORMAT_XRGB8888.into(),
+                    ];
+                    let mut final_formats = vec![];
+
+                    for &f in &possible_formats {
+                        let Ok(vk_fmt) = fourcc_to_vk(f) else {
+                            continue;
+                        };
+                        let Ok(props) = graphics.device.physical_device().format_properties(vk_fmt)
+                        else {
+                            continue;
+                        };
+                        final_formats.push(DrmFormat {
+                            fourcc: f,
+                            modifiers: props
+                                .drm_format_modifier_properties
+                                .iter()
+                                // important bit: only allow single-plane
+                                .filter(|m| m.drm_format_modifier_plane_count == 1)
+                                .map(|m| m.drm_format_modifier)
+                                .collect(),
+                        })
+                    }
+                    final_formats
+                }
+            });
+
+            self.capture.init(drm_formats);
+            self.capture.request_new_frame();
+        };
+
+        for frame in self.capture.receive().into_iter() {
+            match frame {
+                WlxFrame::Dmabuf(frame) => {
+                    if !frame.is_valid() {
+                        log::error!("Invalid frame");
+                        continue;
+                    }
+                    match app.graphics.dmabuf_texture(frame) {
+                        Ok(new) => {
+                            let view = ImageView::new_default(new.clone())?;
+
+                            self.last_view = Some(view);
+                        }
+                        Err(e) => {
+                            log::error!(
+                                "{}: Failed to create DMA-buf texture: {}",
+                                self.name,
+                                e.to_string()
+                            );
+                        }
+                    }
+                }
+                WlxFrame::MemFd(frame) => {
+                    let mut upload = app
+                        .graphics
+                        .create_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+
+                    let Some(fd) = frame.plane.fd else {
+                        log::error!("No fd");
+                        continue;
+                    };
+                    log::debug!("{}: New MemFd frame", self.name);
+                    let format = fourcc_to_vk(frame.format.fourcc)?;
+
+                    let len = frame.plane.stride as usize * frame.format.height as usize;
+                    let offset = frame.plane.offset as i64;
+
+                    let map = unsafe {
+                        libc::mmap(
+                            ptr::null_mut(),
+                            len,
+                            libc::PROT_READ,
+                            libc::MAP_SHARED,
+                            fd,
+                            offset,
+                        )
+                    } as *const u8;
+
+                    let data = unsafe { slice::from_raw_parts(map, len) };
+
+                    let image =
+                        upload.texture2d(frame.format.width, frame.format.height, format, data)?;
+                    upload.build_and_execute_now()?;
+
+                    unsafe { libc::munmap(map as *mut _, len) };
+
+                    self.last_view = Some(ImageView::new_default(image)?);
+                    self.capture.request_new_frame();
+                }
+                WlxFrame::MemPtr(frame) => {
+                    log::debug!("{}: New MemPtr frame", self.name);
+                    let mut upload = app
+                        .graphics
+                        .create_command_buffer(CommandBufferUsage::OneTimeSubmit)?;
+
+                    let format = fourcc_to_vk(frame.format.fourcc)?;
+
+                    let data = unsafe { slice::from_raw_parts(frame.ptr as *const u8, frame.size) };
+
+                    let image =
+                        upload.texture2d(frame.format.width, frame.format.height, format, data)?;
+
+                    let pipeline = Some(match self.pipeline {
+                        Some(ref mut p) => p,
+                        _ => {
+                            let mut pipeline = ScreenPipeline::new(&self.extent, app)?;
+                            self.last_view = Some(pipeline.view.clone());
+                            pipeline.ensure_mouse_initialized(&mut upload)?;
+                            self.pipeline = Some(pipeline);
+                            self.pipeline.as_mut().unwrap() // safe
+                        }
+                    });
+
+                    upload.build_and_execute_now()?;
+
+                    if let Some(pipeline) = pipeline {
+                        pipeline.render(image, frame.mouse.as_ref(), app)?;
+                    } else {
+                        let view = ImageView::new_default(image)?;
+                        self.last_view = Some(view);
+                    }
+                    self.capture.request_new_frame();
+                }
+            };
+        }
+        Ok(())
+    }
+    fn pause(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        self.capture.pause();
+        Ok(())
+    }
+    fn resume(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
+        self.capture.resume();
+        Ok(())
+    }
+    fn view(&mut self) -> Option<Arc<ImageView>> {
+        self.last_view.clone()
+    }
+}
+
+#[cfg(feature = "wayland")]
+/// Panics if id is not a valid output id
+fn try_create_screen<O>(
+    wl: &WlxClient,
+    id: u32,
+    pw_token_store: &mut HashMap<String, String>,
+    session: &AppSession,
+) -> Option<OverlayData<O>>
+where
+    O: Default,
+{
+    let output = &wl.outputs.get(id).unwrap(); // safe due to contract
+    log::info!(
+        "{}: Res {}x{} Size {:?} Pos {:?}",
+        output.name,
+        output.size.0,
+        output.size.1,
+        output.logical_size,
+        output.logical_pos,
+    );
+
+    let mut capture: Option<ScreenRenderer> = None;
+
+    if (&*session.config.capture_method == "auto" || &*session.config.capture_method == "dmabuf")
+        && wl.maybe_wlr_dmabuf_mgr.is_some()
+    {
+        log::info!("{}: Using Wlr DMA-Buf", &output.name);
+        capture = ScreenRenderer::new_wlr_dmabuf(output);
+    }
+
+    if &*session.config.capture_method == "screencopy" && wl.maybe_wlr_screencopy_mgr.is_some() {
+        log::info!("{}: Using Wlr Screencopy Wl-SHM", &output.name);
+        capture = ScreenRenderer::new_wlr_screencopy(output);
+    }
+
+    if capture.is_none() {
+        log::info!("{}: Using Pipewire capture", &output.name);
+
+        let display_name = output.name.deref();
+
+        // Find existing token by display
+        let token = pw_token_store.get(display_name).map(|s| s.as_str());
+
+        if let Some(t) = token {
+            log::info!(
+                "Found existing Pipewire token for display {}: {}",
+                display_name,
+                t
+            );
+        }
+
+        match ScreenRenderer::new_pw(output, token, session) {
+            Ok((renderer, restore_token)) => {
+                capture = Some(renderer);
+
+                if let Some(token) = restore_token {
+                    if pw_token_store
+                        .insert(String::from(display_name), token.clone())
+                        .is_none()
+                    {
+                        log::info!("Adding Pipewire token {}", token);
+                    }
+                }
+            }
+            Err(e) => {
+                log::warn!(
+                    "{}: Failed to create Pipewire capture: {:?}",
+                    &output.name,
+                    e
+                );
+            }
+        }
+    }
+    if let Some(capture) = capture {
+        let backend = Box::new(SplitOverlayBackend {
+            renderer: Box::new(capture),
+            interaction: Box::new(ScreenInteractionHandler::new(
+                vec2(output.logical_pos.0 as f32, output.logical_pos.1 as f32),
+                vec2(output.logical_size.0 as f32, output.logical_size.1 as f32),
+                output.transform.into(),
+            )),
+        });
+
+        let axis = Vec3::new(0., 0., 1.);
+
+        let transform = output.transform.into();
+
+        let angle = if session.config.upright_screen_fix {
+            match transform {
+                Transform::_90 | Transform::Flipped90 => PI / 2.,
+                Transform::_180 | Transform::Flipped180 => PI,
+                Transform::_270 | Transform::Flipped270 => -PI / 2.,
+                _ => 0.,
+            }
+        } else {
+            0.
+        };
+
+        let center = Vec2 { x: 0.5, y: 0.5 };
+        let interaction_transform = match transform {
+            Transform::_90 | Transform::Flipped90 => Affine2::from_cols(
+                Vec2::NEG_Y * (output.size.0 as f32 / output.size.1 as f32),
+                Vec2::NEG_X,
+                center,
+            ),
+            Transform::_180 | Transform::Flipped180 => Affine2::from_cols(
+                Vec2::NEG_X,
+                Vec2::NEG_Y * (-output.size.0 as f32 / output.size.1 as f32),
+                center,
+            ),
+            Transform::_270 | Transform::Flipped270 => Affine2::from_cols(
+                Vec2::Y * (output.size.0 as f32 / output.size.1 as f32),
+                Vec2::X,
+                center,
+            ),
+            _ => Affine2::from_cols(
+                Vec2::X,
+                Vec2::Y * (-output.size.0 as f32 / output.size.1 as f32),
+                center,
+            ),
+        };
+
+        Some(OverlayData {
+            state: OverlayState {
+                name: output.name.clone(),
+                show_hide: session
+                    .config
+                    .show_screens
+                    .iter()
+                    .any(|s| s.as_ref() == output.name.as_ref()),
+                grabbable: true,
+                recenter: true,
+                interactable: true,
+                spawn_scale: 1.5 * session.config.desktop_view_scale,
+                spawn_point: vec3a(0., 0.5, -1.),
+                spawn_rotation: Quat::from_axis_angle(axis, angle),
+                interaction_transform,
+                ..Default::default()
+            },
+            backend,
+            ..Default::default()
+        })
+    } else {
+        log::warn!("{}: Will not be used", &output.name);
+        None
+    }
+}
+
+#[derive(Deserialize, Serialize, Default)]
+pub struct TokenConf {
+    #[serde(default = "def_pw_tokens")]
+    pub pw_tokens: Vec<(String, String)>,
+}
+
+#[cfg(feature = "wayland")]
+fn get_pw_token_path() -> PathBuf {
+    let mut path = config_io::get_conf_d_path();
+    path.push("pw_tokens.yaml");
+    path
+}
+
+#[cfg(feature = "wayland")]
+pub fn save_pw_token_config(tokens: &HashMap<String, String>) -> Result<(), Box<dyn Error>> {
+    let mut conf = TokenConf::default();
+
+    for (name, token) in tokens {
+        conf.pw_tokens.push((name.clone(), token.clone()));
+    }
+
+    let yaml = serde_yaml::to_string(&conf)?;
+    std::fs::write(get_pw_token_path(), yaml)?;
+
+    Ok(())
+}
+
+#[cfg(feature = "wayland")]
+pub fn load_pw_token_config() -> Result<HashMap<String, String>, Box<dyn Error>> {
+    let mut map: HashMap<String, String> = HashMap::new();
+
+    let yaml = std::fs::read_to_string(get_pw_token_path())?;
+    let conf: TokenConf = serde_yaml::from_str(yaml.as_str())?;
+
+    for (name, token) in conf.pw_tokens {
+        map.insert(name, token);
+    }
+
+    Ok(map)
+}
+
+#[cfg(not(feature = "wayland"))]
+pub fn get_screens_wayland<O>(_session: &AppSession) -> anyhow::Result<(Vec<OverlayData<O>>, Vec2)>
+where
+    O: Default,
+{
+    anyhow::bail!("Wayland support not enabled")
+}
+
+#[cfg(feature = "wayland")]
+pub fn get_screens_wayland<O>(session: &AppSession) -> anyhow::Result<(Vec<OverlayData<O>>, Vec2)>
+where
+    O: Default,
+{
+    let mut overlays = vec![];
+    let mut wl = WlxClient::new().ok_or_else(|| anyhow::anyhow!("Failed to connect to Wayland"))?;
+
+    // Load existing Pipewire tokens from file
+    let mut pw_tokens: HashMap<String, String> = if let Ok(conf) = load_pw_token_config() {
+        conf
+    } else {
+        HashMap::new()
+    };
+
+    let pw_tokens_copy = pw_tokens.clone();
+
+    let mut origin = (i32::MAX, i32::MAX);
+    for output in wl.outputs.values() {
+        origin.0 = origin.0.min(output.logical_pos.0);
+        origin.1 = origin.1.min(output.logical_pos.1);
+    }
+
+    log::info!("Desktop origin: {:?}", origin);
+
+    // adjust all outputs so that the top-left corner is at (0, 0)
+    for output in wl.outputs.values_mut() {
+        output.logical_pos.0 -= origin.0;
+        output.logical_pos.1 -= origin.1;
+    }
+
+    for id in wl.outputs.keys() {
+        if let Some(overlay) = try_create_screen(&wl, *id, &mut pw_tokens, session) {
+            overlays.push(overlay);
+        }
+    }
+
+    if pw_tokens_copy != pw_tokens {
+        // Token list changed, re-create token config file
+        if let Err(err) = save_pw_token_config(&pw_tokens) {
+            log::error!("Failed to save Pipewire token config: {}", err);
+        }
+    }
+
+    let extent = wl.get_desktop_extent();
+    Ok((overlays, Vec2::new(extent.0 as f32, extent.1 as f32)))
+}
+
+#[cfg(not(feature = "x11"))]
+pub fn get_screens_x11<O>(_session: &AppSession) -> anyhow::Result<(Vec<OverlayData<O>>, Vec2)>
+where
+    O: Default,
+{
+    anyhow::bail!("X11 support not enabled")
+}
+
+#[cfg(feature = "x11")]
+pub fn get_screens_x11<O>(session: &AppSession) -> anyhow::Result<(Vec<OverlayData<O>>, Vec2)>
+where
+    O: Default,
+{
+    use anyhow::bail;
+
+    let mut extent = vec2(0., 0.);
+
+    let monitors = match XshmCapture::get_monitors() {
+        Ok(m) => m,
+        Err(e) => {
+            bail!(e.to_string());
+        }
+    };
+
+    let overlays = monitors
+        .into_iter()
+        .map(|s| {
+            log::info!(
+                "{}: Res {}x{}, Pos {}x{}",
+                s.name,
+                s.monitor.width(),
+                s.monitor.height(),
+                s.monitor.x(),
+                s.monitor.y()
+            );
+            let size = (s.monitor.width(), s.monitor.height());
+            let capture: ScreenRenderer = ScreenRenderer::new_xshm(s.clone());
+
+            let backend = Box::new(SplitOverlayBackend {
+                renderer: Box::new(capture),
+                interaction: Box::new(ScreenInteractionHandler::new(
+                    vec2(s.monitor.x() as f32, s.monitor.y() as f32),
+                    vec2(s.monitor.width() as f32, s.monitor.height() as f32),
+                    Transform::Normal,
+                )),
+            });
+
+            let interaction_transform = Affine2::from_translation(Vec2 { x: 0.5, y: 0.5 })
+                * Affine2::from_scale(Vec2 {
+                    x: 1.,
+                    y: -size.0 as f32 / size.1 as f32,
+                });
+
+            extent.x = extent.x.max((s.monitor.x() + s.monitor.width()) as f32);
+            extent.y = extent.y.max((s.monitor.y() + s.monitor.height()) as f32);
+            OverlayData {
+                state: OverlayState {
+                    name: s.name.clone(),
+                    show_hide: session
+                        .config
+                        .show_screens
+                        .iter()
+                        .any(|x| x.as_ref() == s.name.as_ref()),
+                    grabbable: true,
+                    recenter: true,
+                    interactable: true,
+                    spawn_scale: 1.5 * session.config.desktop_view_scale,
+                    spawn_point: vec3a(0., 0.5, -1.),
+                    spawn_rotation: Quat::IDENTITY,
+                    interaction_transform,
+                    ..Default::default()
+                },
+                backend,
+                ..Default::default()
+            }
+        })
+        .collect();
+
+    Ok((overlays, extent))
+}
+
+#[allow(unused)]
+pub enum Transform {
+    Normal,
+    _90,
+    _180,
+    _270,
+    Flipped90,
+    Flipped180,
+    Flipped270,
+}
+
+#[cfg(feature = "wayland")]
+impl From<wl_output::Transform> for Transform {
+    fn from(t: wl_output::Transform) -> Transform {
+        match t {
+            wl_output::Transform::Normal => Transform::Normal,
+            wl_output::Transform::_90 => Transform::_90,
+            wl_output::Transform::_180 => Transform::_180,
+            wl_output::Transform::_270 => Transform::_270,
+            wl_output::Transform::Flipped => Transform::Flipped180,
+            wl_output::Transform::Flipped90 => Transform::Flipped90,
+            wl_output::Transform::Flipped180 => Transform::Flipped180,
+            wl_output::Transform::Flipped270 => Transform::Flipped270,
+            _ => Transform::Normal,
+        }
+    }
+}
+
+fn extent_from_res(res: (i32, i32)) -> [u32; 3] {
+    // screens above a certain resolution will have severe aliasing
+
+    // TODO make dynamic. maybe don't go above HMD resolution?
+    let w = res.0.min(2560) as u32;
+    let h = (res.1 as f32 / res.0 as f32 * w as f32) as u32;
+    [w, h, 1]
+}

src/overlays/toast.rs

@@ -0,0 +1,153 @@
+use std::{
+    ops::Add,
+    sync::{atomic::AtomicUsize, Arc},
+    time::Instant,
+};
+
+use glam::vec3a;
+
+use crate::{
+    backend::{
+        common::{OverlaySelector, TaskType},
+        overlay::{OverlayBackend, OverlayState, RelativeTo},
+    },
+    gui::{color_parse, CanvasBuilder},
+    state::AppState,
+};
+
+const FONT_SIZE: isize = 16;
+const PADDING: (f32, f32) = (25., 7.);
+const PIXELS_TO_METERS: f32 = 1. / 2000.;
+const TOAST_AUDIO_WAV: &[u8] = include_bytes!("../res/557297.wav");
+
+static AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(0);
+
+pub struct Toast {
+    pub title: Arc<str>,
+    pub body: Arc<str>,
+    pub opacity: f32,
+    pub timeout: f32,
+    pub sound: bool,
+}
+
+#[allow(dead_code)]
+impl Toast {
+    pub fn new(title: Arc<str>, body: Arc<str>) -> Self {
+        Toast {
+            title,
+            body,
+            opacity: 1.0,
+            timeout: 3.0,
+            sound: false,
+        }
+    }
+    pub fn with_timeout(mut self, timeout: f32) -> Self {
+        self.timeout = timeout;
+        self
+    }
+    pub fn with_opacity(mut self, opacity: f32) -> Self {
+        self.opacity = opacity;
+        self
+    }
+    pub fn with_sound(mut self, sound: bool) -> Self {
+        self.sound = sound;
+        self
+    }
+    pub fn submit(self, app: &mut AppState) {
+        self.submit_at(app, Instant::now());
+    }
+    pub fn submit_at(self, app: &mut AppState, instant: Instant) {
+        let auto_increment = AUTO_INCREMENT.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
+        let name: Arc<str> = format!("toast-{}", auto_increment).into();
+        let selector = OverlaySelector::Name(name.clone());
+
+        let destroy_at = instant.add(std::time::Duration::from_secs_f32(self.timeout));
+
+        let has_sound = self.sound && app.session.config.notifications_sound_enabled;
+
+        app.tasks.enqueue_at(
+            TaskType::CreateOverlay(
+                selector.clone(),
+                Box::new(move |app| new_toast(self, name, app)),
+            ),
+            instant,
+        );
+
+        app.tasks
+            .enqueue_at(TaskType::DropOverlay(selector), destroy_at);
+
+        if has_sound {
+            app.audio.play(TOAST_AUDIO_WAV);
+        }
+    }
+}
+
+fn new_toast(
+    toast: Toast,
+    name: Arc<str>,
+    app: &mut AppState,
+) -> Option<(OverlayState, Box<dyn OverlayBackend>)> {
+    let title = if toast.title.len() > 0 {
+        toast.title
+    } else {
+        "Notification".into()
+    };
+
+    let mut size = if toast.body.len() > 0 {
+        let (w0, _) = app
+            .fc
+            .get_text_size(&title, FONT_SIZE, app.graphics.clone())
+            .ok()?;
+        let (w1, h1) = app
+            .fc
+            .get_text_size(&toast.body, FONT_SIZE, app.graphics.clone())
+            .ok()?;
+        (w0.max(w1), h1 + 50.)
+    } else {
+        let (w, h) = app
+            .fc
+            .get_text_size(&title, FONT_SIZE, app.graphics.clone())
+            .ok()?;
+        (w, h + 20.)
+    };
+
+    let og_width = size.0;
+    size.0 += PADDING.0 * 2.;
+
+    let mut canvas = CanvasBuilder::<(), ()>::new(
+        size.0 as _,
+        size.1 as _,
+        app.graphics.clone(),
+        app.graphics.native_format,
+        (),
+    )
+    .ok()?;
+
+    canvas.font_size = FONT_SIZE;
+    canvas.fg_color = color_parse("#aaaaaa").unwrap(); // want panic
+    canvas.bg_color = color_parse("#333333").unwrap(); // want panic
+    canvas.panel(0., 0., size.0, size.1);
+
+    if toast.body.len() > 0 {
+        canvas.label(PADDING.0, 54., og_width, size.1 - 54., toast.body);
+
+        canvas.fg_color = color_parse("#101010").unwrap(); // want panic
+        canvas.bg_color = color_parse("#666666").unwrap(); // want panic
+        canvas.panel(0., 0., size.0, 30.);
+        canvas.label_centered(PADDING.0, 16., og_width, FONT_SIZE as f32 + 2., title);
+    } else {
+        canvas.label_centered(PADDING.0, 0., og_width, size.1, title);
+    }
+
+    let state = OverlayState {
+        name,
+        want_visible: true,
+        spawn_scale: size.0 * PIXELS_TO_METERS,
+        spawn_point: vec3a(0., -0.2, -0.5),
+        relative_to: RelativeTo::Head,
+        ..Default::default()
+    };
+    let backend = Box::new(canvas.build());
+
+    Some((state, backend))
+}

src/overlays/watch.rs

@@ -0,0 +1,109 @@
+use glam::{Quat, Vec3A};
+use serde::{Deserialize, Serialize};
+use std::path::PathBuf;
+
+use crate::{
+    backend::overlay::{ui_transform, OverlayData, OverlayState, RelativeTo},
+    config::{
+        def_half, def_left, def_point7, def_watch_pos, def_watch_rot, load_known_yaml, ConfigType,
+    },
+    config_io,
+    gui::modular::{modular_canvas, ModularUiConfig},
+    state::{AppState, LeftRight},
+};
+
+pub const WATCH_NAME: &str = "watch";
+
+pub fn create_watch<O>(state: &AppState) -> anyhow::Result<OverlayData<O>>
+where
+    O: Default,
+{
+    let config = load_known_yaml::<ModularUiConfig>(ConfigType::Watch);
+
+    let canvas = modular_canvas(&config.size, &config.elements, state)?;
+
+    let relative_to = RelativeTo::Hand(state.session.config.watch_hand as usize);
+
+    Ok(OverlayData {
+        state: OverlayState {
+            name: WATCH_NAME.into(),
+            want_visible: true,
+            interactable: true,
+            spawn_scale: config.width,
+            spawn_point: Vec3A::from_slice(&state.session.config.watch_pos),
+            spawn_rotation: Quat::from_slice(&state.session.config.watch_rot),
+            interaction_transform: ui_transform(&config.size),
+            relative_to,
+            ..Default::default()
+        },
+        backend: Box::new(canvas),
+        ..Default::default()
+    })
+}
+
+pub fn watch_fade<D>(app: &mut AppState, watch: &mut OverlayData<D>)
+where
+    D: Default,
+{
+    if watch.state.saved_scale.is_some_and(|s| s < f32::EPSILON) {
+        watch.state.want_visible = false;
+        return;
+    }
+
+    let to_hmd = (watch.state.transform.translation - app.input_state.hmd.translation).normalize();
+    let watch_normal = watch
+        .state
+        .transform
+        .transform_vector3a(Vec3A::NEG_Z)
+        .normalize();
+    let dot = to_hmd.dot(watch_normal);
+
+    if dot < app.session.config.watch_view_angle_min {
+        watch.state.want_visible = false;
+    } else {
+        watch.state.want_visible = true;
+
+        watch.state.alpha = (dot - app.session.config.watch_view_angle_min)
+            / (app.session.config.watch_view_angle_max - app.session.config.watch_view_angle_min);
+        watch.state.alpha += 0.1;
+        watch.state.alpha = watch.state.alpha.clamp(0., 1.);
+    }
+}
+
+#[derive(Deserialize, Serialize)]
+pub struct WatchConf {
+    #[serde(default = "def_watch_pos")]
+    pub watch_pos: [f32; 3],
+
+    #[serde(default = "def_watch_rot")]
+    pub watch_rot: [f32; 4],
+
+    #[serde(default = "def_left")]
+    pub watch_hand: LeftRight,
+
+    #[serde(default = "def_half")]
+    pub watch_view_angle_min: f32,
+
+    #[serde(default = "def_point7")]
+    pub watch_view_angle_max: f32,
+}
+
+fn get_config_path() -> PathBuf {
+    let mut path = config_io::get_conf_d_path();
+    path.push("watch_state.yaml");
+    path
+}
+pub fn save_watch(app: &mut AppState) -> anyhow::Result<()> {
+    let conf = WatchConf {
+        watch_pos: app.session.config.watch_pos,
+        watch_rot: app.session.config.watch_rot,
+        watch_hand: app.session.config.watch_hand,
+        watch_view_angle_min: app.session.config.watch_view_angle_min,
+        watch_view_angle_max: app.session.config.watch_view_angle_max,
+    };
+
+    let yaml = serde_yaml::to_string(&conf)?;
+    std::fs::write(get_config_path(), yaml)?;
+
+    Ok(())
+}

src/res/actions.json

@@ -0,0 +1,88 @@
+{
+  "actions": [
+    {
+      "name": "/actions/default/in/Click",
+      "type": "boolean",
+      "requirement": "mandatory"
+    },
+    {
+      "name": "/actions/default/in/Grab",
+      "type": "boolean",
+      "requirement": "mandatory"
+    },
+    {
+      "name": "/actions/default/in/Scroll",
+      "type": "vector2",
+      "requirement": "mandatory"
+    },
+    {
+      "name": "/actions/default/in/ShowHide",
+      "type": "boolean",
+      "requirement": "mandatory"
+    },
+    {
+      "name": "/actions/default/in/AltClick",
+      "type": "boolean",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/in/ClickModifierRight",
+      "type": "boolean",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/in/ClickModifierMiddle",
+      "type": "boolean",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/in/SpaceDrag",
+      "type": "boolean",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/in/SpaceRotate",
+      "type": "boolean",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/in/LeftHand",
+      "type": "pose",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/in/RightHand",
+      "type": "pose",
+      "requirement": "optional"
+    },
+    {
+      "name": "/actions/default/out/HapticsLeft",
+      "type": "vibration"
+    },
+    {
+      "name": "/actions/default/out/HapticsRight",
+      "type": "vibration"
+    }
+  ],
+  "action_sets": [
+    {
+      "name": "/actions/default",
+      "usage": "leftright"
+    }
+  ],
+  "default_bindings": [
+    {
+      "controller_type": "knuckles",
+      "binding_url": "actions_binding_knuckles.json"
+    },
+    {
+      "controller_type": "oculus_touch",
+      "binding_url": "actions_binding_oculus.json"
+    },
+    {
+      "controller_type": "vive_controller",
+      "binding_url": "actions_binding_vive.json"
+    }
+  ],
+  "localization": []
+}

src/res/actions_binding_knuckles.json

@@ -0,0 +1,173 @@
+{
+   "action_manifest_version" : 0,
+   "app_key" : "galister.wlxoverlay-s",
+   "bindings" : {
+      "/actions/default" : {
+         "haptics" : [
+            {
+               "output" : "/actions/default/out/hapticsleft",
+               "path" : "/user/hand/left/output/haptic"
+            },
+            {
+               "output" : "/actions/default/out/hapticsright",
+               "path" : "/user/hand/right/output/haptic"
+            }
+         ],
+         "poses" : [
+            {
+               "output" : "/actions/default/in/lefthand",
+               "path" : "/user/hand/left/pose/tip"
+            },
+            {
+               "output" : "/actions/default/in/righthand",
+               "path" : "/user/hand/right/pose/tip"
+            }
+         ],
+         "sources" : [
+            {
+               "inputs" : {
+                  "double" : {
+                     "output" : "/actions/default/in/showhide"
+                  },
+                  "touch" : {
+                     "output": "/actions/default/in/clickmodifierright"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/left/input/b"
+            },
+            {
+               "path": "/user/hand/left/input/a",
+               "mode": "button",
+               "inputs": {
+                  "touch": {
+                     "output": "/actions/default/in/clickmodifiermiddle"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/b",
+               "mode": "button",
+               "inputs": {
+                  "touch": {
+                     "output": "/actions/default/in/clickmodifierright"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/a",
+               "mode": "button",
+               "inputs": {
+                  "touch": {
+                     "output": "/actions/default/in/clickmodifiermiddle"
+                  }
+               }
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/click"
+                  }
+               },
+               "mode" : "button",
+               "parameters" : {
+                  "click_activate_threshold" : "0.35",
+                  "click_deactivate_threshold" : "0.31"
+               },
+               "path" : "/user/hand/left/input/trigger"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/click"
+                  }
+               },
+               "mode" : "button",
+               "parameters" : {
+                  "click_activate_threshold" : "0.35",
+                  "click_deactivate_threshold" : "0.31"
+               },
+               "path" : "/user/hand/right/input/trigger"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/altclick"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/right/input/trackpad"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/spacedrag"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/left/input/trackpad"
+            },
+            {
+               "inputs" : {
+                  "grab" : {
+                     "output" : "/actions/default/in/grab"
+                  }
+               },
+               "mode" : "grab",
+               "parameters" : {
+                  "value_hold_threshold" : "1.3",
+                  "value_release_threshold" : "1.1"
+               },
+               "path" : "/user/hand/left/input/grip"
+            },
+            {
+               "inputs" : {
+                  "grab" : {
+                     "output" : "/actions/default/in/grab"
+                  }
+               },
+               "mode" : "grab",
+               "parameters" : {
+                  "value_hold_threshold" : "1.3",
+                  "value_release_threshold" : "1.1"
+               },
+               "path" : "/user/hand/right/input/grip"
+            },
+            {
+               "inputs" : {
+                  "scroll" : {
+                     "output" : "/actions/default/in/scroll"
+                  }
+               },
+               "mode" : "scroll",
+               "parameters" : {
+                  "scroll_mode" : "smooth"
+               },
+               "path" : "/user/hand/left/input/thumbstick"
+            },
+            {
+               "inputs" : {
+                  "scroll" : {
+                     "output" : "/actions/default/in/scroll"
+                  }
+               },
+               "mode" : "scroll",
+               "parameters" : {
+                  "scroll_mode" : "smooth"
+               },
+               "path" : "/user/hand/right/input/thumbstick"
+            }
+         ]
+      }
+   },
+   "category" : "steamvr_input",
+   "controller_type" : "knuckles",
+   "description" : "Ver1",
+   "interaction_profile" : "",
+   "name" : "WlxOverlay configuration for Index Controller",
+   "options" : {
+      "mirror_actions" : false,
+      "simulated_controller_type" : "none"
+   },
+   "simulated_actions" : []
+}

src/res/actions_binding_oculus.json

@@ -0,0 +1,139 @@
+{
+   "action_manifest_version" : 0,
+   "app_key" : "galister.wlxoverlay-s",
+   "bindings" : {
+      "/actions/default" : {
+         "haptics" : [
+            {
+               "output" : "/actions/default/out/hapticsleft",
+               "path" : "/user/hand/left/output/haptic"
+            },
+            {
+               "output" : "/actions/default/out/hapticsright",
+               "path" : "/user/hand/right/output/haptic"
+            }
+         ],
+         "poses" : [
+            {
+               "output" : "/actions/default/in/lefthand",
+               "path" : "/user/hand/left/pose/tip"
+            },
+            {
+               "output" : "/actions/default/in/righthand",
+               "path" : "/user/hand/right/pose/tip"
+            }
+         ],
+         "sources" : [
+            {
+               "inputs" : {
+                  "double" : {
+                     "output" : "/actions/default/in/showhide"
+                  },
+                  "touch" : {
+                     "output": "/actions/default/in/clickmodifierright"
+                  }
+               },
+               "mode" : "button",
+               "path" : "/user/hand/left/input/y"
+            },
+            {
+               "path": "/user/hand/left/input/x",
+               "mode": "button",
+               "inputs": {
+                  "touch": {
+                     "output": "/actions/default/in/clickmodifiermiddle"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/b",
+               "mode": "button",
+               "inputs": {
+                  "touch": {
+                     "output": "/actions/default/in/clickmodifierright"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/a",
+               "mode": "button",
+               "inputs": {
+                  "touch": {
+                     "output": "/actions/default/in/clickmodifiermiddle"
+                  }
+               }
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/click"
+                  }
+               },
+               "mode": "button",
+               "path" : "/user/hand/left/input/trigger"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/click"
+                  }
+               },
+               "mode": "button",
+               "path" : "/user/hand/right/input/trigger"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/grab"
+                  }
+               },
+               "mode": "button",
+               "path" : "/user/hand/left/input/grip"
+            },
+            {
+               "inputs" : {
+                  "click" : {
+                     "output" : "/actions/default/in/grab"
+                  }
+               },
+               "mode": "button",
+               "path" : "/user/hand/right/input/grip"
+            },
+            {
+               "inputs" : {
+                  "scroll" : {
+                     "output" : "/actions/default/in/scroll"
+                  }
+               },
+               "mode" : "scroll",
+               "parameters" : {
+                  "scroll_mode" : "smooth"
+               },
+               "path" : "/user/hand/left/input/joystick"
+            },
+            {
+               "inputs" : {
+                  "scroll" : {
+                     "output" : "/actions/default/in/scroll"
+                  }
+               },
+               "mode" : "scroll",
+               "parameters" : {
+                  "scroll_mode" : "smooth"
+               },
+               "path" : "/user/hand/right/input/joystick"
+            }
+         ]
+      }
+   },
+   "category" : "steamvr_input",
+   "controller_type" : "oculus_touch",
+   "description" : "Ver1",
+   "interaction_profile" : "",
+   "name" : "WlxOverlay configuration for Oculus Touch Controller",
+   "options" : {
+      "mirror_actions" : false,
+      "simulated_controller_type" : "none"
+   },
+   "simulated_actions" : []
+}

src/res/actions_binding_vive.json

@@ -0,0 +1,139 @@
+{
+   "action_manifest_version" : 0,
+   "app_key" : "galister.wlxoverlay-s",
+   "bindings" : {
+      "/actions/default": {
+         "poses": [
+            {
+               "path": "/user/hand/left/pose/tip",
+               "output": "/actions/default/in/lefthand"
+            },
+            {
+               "path": "/user/hand/right/pose/tip",
+               "output": "/actions/default/in/righthand"
+            }
+         ],
+         "haptics": [
+            {
+               "output": "/actions/default/out/hapticsleft",
+               "path": "/user/hand/left/output/haptic"
+            },
+            {
+               "output": "/actions/default/out/hapticsright",
+               "path": "/user/hand/right/output/haptic"
+            }
+         ],
+         "sources": [
+            {
+               "path": "/user/hand/left/input/grip",
+               "mode": "button",
+               "inputs": {
+                  "click": {
+                     "output": "/actions/default/in/grab"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/grip",
+               "mode": "button",
+               "inputs": {
+                  "click": {
+                     "output": "/actions/default/in/grab"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/left/input/trigger",
+               "mode": "button",
+               "inputs": {
+                  "click": {
+                     "output": "/actions/default/in/click"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/trigger",
+               "mode": "trigger",
+               "inputs": {
+                  "click": {
+                     "output": "/actions/default/in/click"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/trackpad",
+               "mode": "scroll",
+               "parameters": {
+                  "scroll_mode": "discrete"
+               },
+               "inputs": {
+                  "scroll": {
+                     "output": "/actions/default/in/scroll"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/left/input/trackpad",
+               "mode": "scroll",
+               "parameters": {
+                  "scroll_mode": "discrete"
+               },
+               "inputs": {
+                  "scroll": {
+                     "output": "/actions/default/in/scroll"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/left/input/application_menu",
+               "mode": "button",
+               "inputs": {
+                  "double": {
+                     "output": "/actions/default/in/showhide"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/right/input/trackpad",
+               "mode": "dpad",
+               "parameters": {
+                  "sub_mode": "touch"
+               },
+               "inputs": {
+                  "west": {
+                     "output": "/actions/default/in/clickmodifiermiddle"
+                  },
+                  "east": {
+                     "output": "/actions/default/in/clickmodifierright"
+                  }
+               }
+            },
+            {
+               "path": "/user/hand/left/input/trackpad",
+               "mode": "dpad",
+               "parameters": {
+                  "sub_mode": "touch"
+               },
+               "inputs": {
+                  "west": {
+                     "output": "/actions/default/in/clickmodifiermiddle"
+                  },
+                  "east": {
+                     "output": "/actions/default/in/clickmodifierright"
+                  }
+               }
+            }
+         ]
+      }
+   },
+   "category" : "steamvr_input",
+   "controller_type" : "vive_controller",
+   "description" : "Ver1",
+   "interaction_profile" : "",
+   "name" : "WlxOverlay configuration for Vive Controller",
+   "options" : {
+      "mirror_actions" : false,
+      "simulated_controller_type" : "none"
+   },
+   "simulated_actions" : []
+}

src/res/config.yaml

@@ -0,0 +1,13 @@
+# For how much time mouse motion events should be stopped after clicking?
+# Prevents accidental dragging various GUI elements or links, making it easier to click
+# Default: 300
+click_freeze_time_ms: 300
+
+# Default: true
+keyboard_sound_enabled: true
+
+# Alter default scale of various overlays
+# Default: 1.0
+keyboard_scale: 1.0
+desktop_view_scale: 1.0
+watch_scale: 1.0

src/res/keyboard.yaml

@@ -0,0 +1,109 @@
+---
+# looking to make changes? 
+# drop me in ~/.config/wlxoverlay/keyboard.yaml
+
+# This file contains all data needed to generate the keyboard.
+# You can create any layout, as long as:
+# - All keys are rectangular with 1 unit of height.
+# This means:
+# - We're limited to the flat & boring ANSI enter key.
+# - Numpad + and Enter might not look so great.
+
+# *** Important ***
+# The keyboard layout uses virtual key codes, so they are layout-independent.
+# For example, Q on a French layout actually results in A.
+# If you're using a non-english layout, chances are you only need to edit the label section below.
+
+# Not used for anything right now
+name: "en-us_full"
+
+# How many units of key size in each row? 1 = standard letter key size
+row_size: 23
+
+# Specifies the size of each key. The sum of any given row must equal RowSize
+key_sizes:
+    - [1.5,0.5, 1, 1, 1, 1,0.5,1, 1, 1, 1,0.5,1, 1, 1, 1, 0.5, 1, 1, 1, 0.5, 1, 1, 1, 1]
+    - [1,   1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2,     0.5, 1, 1, 1, 0.5,    1, 1, 1, 1]
+    - [1.5,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1.5,  0.5, 1, 1, 1, 0.5,    1, 1, 1, 1]
+    - [1.75,  1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2.25,   4,                    1, 1, 1, 1]
+    - [2.25,    1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2.75,    1.5,    1, 1.5,       1, 1, 1, 1]
+    - [1.25, 1.25, 1.25, 6.25, 1.25, 1.25, 1.25, 1.25, 0.5, 1, 1, 1, 0.5,    2,    1, 1]
+
+# The main (blue) layout of the keyboard.
+# Accepted are:
+# - virtual keys. For a full list, look at enum VirtualKey in https://github.com/galister/wlx-overlay-s/blob/main/src/hid.rs
+# - exec_commands (defined below)
+# - macros (defined below)
+# - ~ (null) will leave an empty space with the corresponding size from key_sizes
+main_layout:
+    - ["Escape", ~, "F1", "F2", "F3", "F4", ~, "F5", "F6", "F7", "F8", ~, "F9", "F10", "F11", "F12", ~, "Print", "Scroll", "Pause", ~, "COPY", "PASTE", ~, "KILL"]
+    - ["Oem3", "N1", "N2", "N3", "N4", "N5", "N6", "N7", "N8", "N9", "N0", "Minus", "Plus", "BackSpace", ~, "Insert", "Home", "Prior", ~, "NumLock", "KP_Divide", "KP_Multiply", "KP_Subtract"]
+    - ["Tab", "Q", "W", "E", "R", "T", "Y", "U", "I", "O", "P", "Oem4", "Oem6", "Oem5", ~, "Delete", "End", "Next", ~, "KP_7", "KP_8", "KP_9", "KP_Add"]
+    - ["XF86Favorites", "A", "S", "D", "F", "G", "H", "J", "K", "L", "Oem1", "Oem7", "Return", ~, "KP_4", "KP_5", "KP_6", ~]
+    - ["LShift", "Z", "X", "C", "V", "B", "N", "M", "Comma", "Period", "Oem2", "RShift", ~, "Up", ~, "KP_1", "KP_2", "KP_3", "KP_Enter"]
+    - ["LCtrl", "LSuper", "LAlt", "Space", "Meta", "RSuper", "Menu", "RCtrl", ~, "Left", "Down", "Right", ~, "KP_0", "KP_Decimal", ~]
+
+# Shell commands to be used in a layout.
+# Value is an array of string arguments.
+exec_commands:
+    STT: [ "whisper_stt", "--lang", "en" ]
+
+# Series of keypresses to be used in a layout.
+# Format: keyName [DOWN|UP]
+# keyName must be a valid virtual key from the VirtualKey enum (see above)
+# DOWN|UP: can be omitted for an implicit "keyName DOWN, keyName UP"
+macros:
+    KILL: [ "LSuper DOWN", "LCtrl DOWN", "Escape", "LCtrl UP", "LSuper UP" ]
+    COPY: [ "LCtrl DOWN", "C", "LCtrl UP" ]
+    PASTE: [ "LCtrl DOWN", "V", "LCtrl UP" ]
+
+# Custom labels to use.
+# Key: element of main_layout
+# Value: Array of strings. 0th element is the upper row, 1st element is lower row.
+# For empty labels, use [] (do not use ~)
+labels:
+    "N1":            ["1", "!"]
+    "N2":            ["2", "@"]
+    "N3":            ["3", "#"]
+    "N4":            ["4", "$"]
+    "N5":            ["5", "%"]
+    "N6":            ["6", "^"]
+    "N7":            ["7", "&"]
+    "N8":            ["8", "*"]
+    "N9":            ["9", "("]
+    "N0":            ["0", ")"]
+    "Escape":       ["Esc"]
+    "Prior":        ["PgUp"]
+    "Next":         ["PgDn"]
+    "NumLock":      ["Num"]
+    "Space":        []
+    "LAlt":         ["Alt"]
+    "LCtrl":        ["Ctrl"]
+    "RCtrl":        ["Ctrl"]
+    "LSuper":       ["Super"]
+    "RSuper":       ["Super"]
+    "LShift":       ["Shift"]
+    "RShift":       ["Shift"]
+    "Insert":       ["Ins"]
+    "Delete":       ["Del"]
+    "Minus":        ["-", "_"]
+    "Plus":         ["=", "+"]
+    "BackSpace":    ["<<"]
+    "Comma":        [" ,",  "<"]
+    "Period":       [" .",  ">"]
+    "Oem1":         [" ;",  ":"]
+    "Oem2":         [" /",  "?"]
+    "Oem3":         ["`", "~"]
+    "Oem4":         [" [",  "{"]
+    "Oem5":         [" \\", "|"]
+    "Oem6":         [" ]",  "}"]
+    "Oem7":         [" '",  "\""]
+    "Oem102":       [" \\", "|"]
+    "KP_Divide":    [" /"]
+    "KP_Add":       [" +"]
+    "KP_Multiply":  [" *"]
+    "KP_Decimal":   [" ."]
+    "KP_Subtract":  [" -"]
+    "KP_Enter":     ["Ent"]
+    "XF86Favorites": ["Rofi"]
+

src/res/settings.yaml

@@ -0,0 +1,560 @@
+# looking to make changes? 
+# drop me in ~/.config/wlxoverlay/settings.yaml
+#
+
+width: 0.3
+
+size: [600, 700]
+
+# +X: right, +Y: up, +Z: back
+spawn_pos: [0, -0.1, -0.5]
+
+elements:
+  - type: Panel
+    rect: [0, 0, 600, 800]
+    bg_color: "#102030"
+
+  - type: Label
+    rect: [15, 35, 600, 70]
+    font_size: 24
+    fg_color: "#ffffff"
+    source: Static
+    text: Settings
+
+  - type: Button
+    rect: [560, 0, 40, 40]
+    font_size: 16
+    bg_color: "#880000"
+    fg_color: "#ffffff"
+    text: X
+    click_down:
+      - type: Window
+        target: "settings"
+        action: Destroy
+
+  - type: Panel
+    rect: [50, 53, 500, 1]
+    bg_color: "#c0c0c0"
+
+  ####### Watch Section #######
+
+  - type: Label
+    rect: [15, 85, 570, 24]
+    font_size: 18
+    fg_color: "#ffffff"
+    source: Static
+    text: Watch
+
+  - type: Panel
+    rect: [250, 105, 1, 100]
+    bg_color: "#c0c0c0"
+
+  - type: Label
+    rect: [288, 105, 100, 24]
+    font_size: 12
+    fg_color: "#ffffff"
+    source: Static
+    text: Visibility
+
+  - type: Button
+    rect: [270, 120, 100, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#306060"
+    text: "Hide"
+    click_down:
+      - type: Watch
+        action: Hide
+
+  - type: Button
+    rect: [270, 170, 100, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#306060"
+    text: "Swap Hand"
+    click_down:
+      - type: Watch
+        action: SwitchHands
+
+  - type: Panel
+    rect: [390, 105, 1, 100]
+    bg_color: "#c0c0c0"
+
+  - type: Label
+    rect: [430, 105, 120, 24]
+    font_size: 12
+    fg_color: "#ffffff"
+    source: Static
+    text: Watch Fade
+
+  - type: Button
+    rect: [410, 120, 140, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#306060"
+    text: "Cutoff Point"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: Watch
+        action:
+          ViewAngle: {kind: "MaxOpacity", delta: 0.01}
+    scroll_down:
+      - type: Watch
+        action:
+          ViewAngle: {kind: "MaxOpacity", delta: -0.01}
+
+  - type: Button
+    rect: [410, 170, 140, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#306060"
+    text: "Cutoff Strength"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: Watch
+        action:
+          ViewAngle: {kind: "MinOpacity", delta: 0.01}
+    scroll_down:
+      - type: Watch
+        action: 
+          ViewAngle: {kind: "MinOpacity", delta: -0.01}
+
+  - type: Label
+    rect: [25, 140, 90, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    source: Static
+    text: Rotation
+
+  - type: Button
+    rect: [108, 120, 30, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#306060"
+    text: "X"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: Watch
+        action:
+          Rotation: {axis: "X", delta: 0.25}
+    scroll_down:
+      - type: Watch
+        action:
+          Rotation: {axis: "X", delta: -0.25}
+
+  - type: Button
+    rect: [153, 120, 30, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#306060"
+    text: "Y"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: Watch
+        action:
+          Rotation: {axis: "Y", delta: 0.25}
+    scroll_down:
+      - type: Watch
+        action:
+          Rotation: {axis: "Y", delta: -0.25}
+
+  - type: Button
+    rect: [198, 120, 30, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#306060"
+    text: "Z"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: Watch
+        action:
+          Rotation: {axis: "Z", delta: 0.25}
+    scroll_down:
+      - type: Watch
+        action:
+          Rotation: {axis: "Z", delta: -0.25}
+
+  - type: Label
+    rect: [25, 190, 90, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    source: Static
+    text: Position
+
+  - type: Button
+    rect: [108, 170, 30, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#306060"
+    text: "X"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: Watch
+        action:
+          Position: {axis: "X", delta: 0.001}
+    scroll_down:
+      - type: Watch
+        action:
+          Position: {axis: "X", delta: -0.001}
+
+  - type: Button
+    rect: [153, 170, 30, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#306060"
+    text: "Y"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: Watch
+        action:
+          Position: {axis: "Y", delta: 0.001}
+    scroll_down:
+      - type: Watch
+        action:
+          Position: {axis: "Y", delta: -0.001}
+
+  - type: Button
+    rect: [198, 170, 30, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#306060"
+    text: "Z"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: Watch
+        action:
+          Position: {axis: "Z", delta: 0.001}
+    scroll_down:
+      - type: Watch
+        action:
+          Position: {axis: "Z", delta: -0.001}
+
+  - type: Panel
+    rect: [50, 220, 500, 1]
+    bg_color: "#c0c0c0"
+
+  ####### Mirror Section #######
+  - type: Label
+    rect: [15, 255, 570, 24]
+    font_size: 18
+    fg_color: "#ffffff"
+    source: Static
+    text: Mirrors
+
+  - type: Label
+    rect: [25, 290, 30, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    source: Static
+    text: M1
+
+  - type: Button
+    rect: [60, 270, 110, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#707070"
+    text: "Show/Hide"
+    click_down: # ToggleVisible if exists, else create
+      - type: Overlay
+        target: M1
+        action: ToggleVisible # only fires if overlay exists
+      - type: Window
+        target: M1
+        action: ShowMirror # only fires if not exists
+
+  - type: Button
+    rect: [185, 270, 60, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#707070"
+    text: "Lock"
+    click_down:
+      - type: Overlay
+        target: M1
+        action: ToggleInteraction
+
+  - type: Button
+    rect: [258, 270, 30, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#880000"
+    text: "X"
+    click_down:
+      - type: Window
+        target: M1
+        action: Destroy
+
+  - type: Label
+    rect: [25, 340, 30, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    source: Static
+    text: M2
+
+  - type: Button
+    rect: [60, 320, 110, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#707070"
+    text: "Show/Hide"
+    click_down:
+      - type: Overlay
+        target: M2
+        action: ToggleVisible
+      - type: Window
+        target: M2
+        action: ShowMirror
+
+  - type: Button
+    rect: [185, 320, 60, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#707070"
+    text: "Lock"
+    click_down:
+      - type: Overlay
+        target: M2
+        action: ToggleInteraction
+
+  - type: Button
+    rect: [258, 320, 30, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#880000"
+    text: "X"
+    click_down:
+      - type: Window
+        target: M2
+        action: Destroy
+
+  - type: Label
+    rect: [25, 390, 30, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    source: Static
+    text: M3
+
+  - type: Button
+    rect: [60, 370, 110, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#707070"
+    text: "Show/Hide"
+    click_down:
+      - type: Overlay
+        target: M3
+        action: ToggleVisible
+      - type: Window
+        target: M3
+        action: ShowMirror
+
+  - type: Button
+    rect: [185, 370, 60, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#707070"
+    text: "Lock"
+    click_down:
+      - type: Overlay
+        target: M3
+        action: ToggleInteraction
+
+  - type: Button
+    rect: [258, 370, 30, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#880000"
+    text: "X"
+    click_down:
+      - type: Window
+        target: M3
+        action: Destroy
+
+  - type: Panel
+    rect: [300, 240, 1, 200]
+    bg_color: "#c0c0c0"
+
+  ####### Color Gain Section #######
+
+  - type: Label
+    rect: [325, 255, 90, 24]
+    font_size: 18
+    fg_color: "#ffffff"
+    source: Static
+    text: Color Gain
+
+  - type: Label
+    rect: [470, 255, 90, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    source: Static
+    text: (SteamVR)
+
+  - type: Button
+    rect: [330, 270, 60, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#707070"
+    text: "All"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: ColorAdjust
+        channel: All
+        delta: 0.01
+    scroll_down:
+      - type: ColorAdjust
+        channel: All
+        delta: -0.01
+
+  - type: Button
+    rect: [405, 270, 30, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#701010"
+    text: "R"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: ColorAdjust
+        channel: R
+        delta: 0.01
+    scroll_down:
+      - type: ColorAdjust
+        channel: R
+        delta: -0.01
+
+  - type: Button
+    rect: [450, 270, 30, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#107010"
+    text: "G"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: ColorAdjust
+        channel: G
+        delta: 0.01
+    scroll_down:
+      - type: ColorAdjust
+        channel: G
+        delta: -0.01
+
+  - type: Button
+    rect: [495, 270, 30, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#101070"
+    text: "B"
+    click_down:
+      - type: Toast
+        message: Use stick up/down while hovering the button!
+    scroll_up:
+      - type: ColorAdjust
+        channel: B
+        delta: 0.01
+    scroll_down:
+      - type: ColorAdjust
+        channel: B
+        delta: -0.01
+
+  - type: Panel
+    rect: [325, 315, 225, 1]
+    bg_color: "#c0c0c0"
+
+  ####### Playspace Section #######
+
+  - type: Label
+    rect: [325, 345, 90, 24]
+    font_size: 18
+    fg_color: "#ffffff"
+    source: Static
+    text: Playspace
+
+  - type: Button
+    rect: [330, 360, 220, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#206060"
+    text: "Fix Floor"
+    click_down:
+      - type: System
+        action: PlayspaceFixFloor
+
+  - type: Button
+    rect: [330, 410, 220, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#206060"
+    text: "Reset Offset"
+    click_down:
+      - type: System
+        action: PlayspaceResetOffset
+
+  ####### Notifications Section #######
+
+  - type: Panel
+    rect: [50, 460, 500, 1]
+    bg_color: "#c0c0c0"
+
+  - type: Label
+    rect: [325, 490, 90, 24]
+    font_size: 18
+    fg_color: "#ffffff"
+    source: Static
+    text: Notifications
+
+  - type: Button
+    rect: [330, 505, 220, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#606020"
+    text: "Enabled"
+    click_down:
+      - type: System
+        action: ToggleNotifications
+
+  - type: Button
+    rect: [330, 555, 220, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#606020"
+    text: "Sound Enabled"
+    click_down:
+      - type: System
+        action: ToggleNotificationSounds
+
+  - type: Panel
+    rect: [50, 605, 500, 1]
+    bg_color: "#c0c0c0"
+
+  - type: Button
+    rect: [330, 625, 220, 30]
+    font_size: 12
+    fg_color: "#ffffff"
+    bg_color: "#206060"
+    text: "Save Config"
+    click_down:
+      - type: System
+        action: PersistConfig
+

src/res/watch.yaml

@@ -0,0 +1,166 @@
+# looking to make changes? 
+# drop me in ~/.config/wlxoverlay/watch.yaml
+#
+
+width: 0.115
+
+size: [400, 200]
+
+elements:
+  # background panel
+  - type: Panel
+    rect: [0, 0, 400, 200]
+    bg_color: "#353535"
+
+  - type: Button
+    rect: [2, 162, 26, 36]
+    font_size: 14
+    bg_color: "#808040"
+    fg_color: "#ffffff"
+    text: "C"
+    click_up: # destroy if exists, otherwise create
+      - type: Window
+        target: settings
+        action: ShowUi # only triggers if not exists
+      - type: Window 
+        target: settings
+        action: Destroy # only triggers if exists since before current frame
+
+  # Keyboard button
+  - type: Button
+    rect: [32, 162, 60, 36]
+    font_size: 14
+    fg_color: "#FFFFFF"
+    bg_color: "#406050"
+    text: Kbd
+    click_up:
+      - type: Overlay
+        target: "kbd"
+        action: ToggleVisible
+    long_click_up:
+      - type: Overlay
+        target: "kbd"
+        action: Reset
+    right_up:
+      - type: Overlay
+        target: "kbd"
+        action: ToggleImmovable
+    middle_up:
+      - type: Overlay
+        target: "kbd"
+        action: ToggleInteraction
+    scroll_up:
+      - type: Overlay
+        target: "kbd"
+        action: 
+          Opacity: { delta: 0.025 }
+    scroll_down:
+      - type: Overlay
+        target: "kbd"
+        action: 
+          Opacity: { delta: -0.025 }
+
+  # bottom row, of keyboard + overlays
+  - type: OverlayList
+    rect: [94, 160, 306, 40]
+    font_size: 14
+    fg_color: "#FFFFFF"
+    bg_color: "#405060"
+    layout: Horizontal
+    click_up: ToggleVisible
+    long_click_up: Reset
+    right_up: ToggleImmovable
+    middle_up: ToggleInteraction
+    scroll_up: 
+      Opacity: { delta: 0.025 }
+    scroll_down: 
+      Opacity: { delta: -0.025 }
+
+  # local clock
+  - type: Label
+    rect: [19, 90, 200, 50]
+    font_size: 46
+    fg_color: "#ffffff"
+    source: Clock
+    format: "%H:%M" # 23:59
+    #format: "%h:%M %p" # 11:59 PM
+ 
+  # local date
+  - type: Label
+    rect: [20, 117, 200, 20]
+    font_size: 14
+    fg_color: "#ffffff"
+    source: Clock
+    format: "%x" # local date representation
+
+  # local day-of-week
+  - type: Label
+    rect: [20, 137, 200, 50]
+    font_size: 14
+    fg_color: "#ffffff"
+    source: Clock
+    format: "%A" # Tuesday
+    #format: "%a" # Tue
+
+  # alt clock 1
+  - type: Label
+    rect: [210, 90, 200, 50]
+    font_size: 24
+    fg_color: "#99BBAA"
+    source: Clock
+    timezone: "Asia/Tokyo" # change TZ1 here
+    format: "%H:%M" # 23:59
+    #format: "%h:%M %p" # 11:59 PM
+  - type: Label
+    rect: [210, 60, 200, 50]
+    font_size: 14
+    fg_color: "#99BBAA"
+    source: Static
+    text: "Tokyo" # change TZ1 label here
+ 
+  # alt clock 2
+  - type: Label
+    rect: [210, 150, 200, 50]
+    font_size: 24
+    fg_color: "#AA99BB"
+    source: Clock
+    timezone: "America/Chicago" # change TZ2 here
+    format: "%H:%M" # 23:59
+    #format: "%h:%M %p" # 11:59 PM
+  - type: Label
+    rect: [210, 120, 200, 50]
+    font_size: 14
+    fg_color: "#AA99BB"
+    source: Static
+    text: "Chicago" # change TZ2 label here
+
+  # batteries
+  - type: BatteryList
+    rect: [0, 0, 400, 30]
+    font_size: 14
+    fg_color: "#99BBAA"
+    fg_color_low: "#B06060"
+    fg_color_charging: "#6080A0"
+    num_devices: 9
+    layout: Horizontal
+    low_threshold: 20
+
+  # volume buttons
+  - type: Button
+    rect: [327, 52, 46, 32]
+    font_size: 14
+    fg_color: "#FFFFFF"
+    bg_color: "#505050"
+    text: "+"
+    click_down:
+      - type: Exec
+        command: [ "pactl", "set-sink-volume", "@DEFAULT_SINK@", "+5%" ]
+  - type: Button
+    rect: [327, 116, 46, 32]
+    font_size: 14
+    fg_color: "#FFFFFF"
+    bg_color: "#505050"
+    text: "-"
+    click_down:
+      - type: Exec
+        command: [ "pactl", "set-sink-volume", "@DEFAULT_SINK@", "-5%" ]

src/shaders/mod.rs

@@ -0,0 +1,137 @@
+pub mod vert_common {
+    vulkano_shaders::shader! {
+        ty: "vertex",
+        src: r"#version 310 es
+            precision highp float;
+
+            layout (location = 0) in vec2 in_pos;
+            layout (location = 1) in vec2 in_uv;
+            layout (location = 0) out vec2 out_uv;
+
+            void main() {
+                out_uv = in_uv;
+                gl_Position = vec4(in_pos * 2. - 1., 0., 1.);
+            }
+        ",
+    }
+}
+
+pub mod frag_color {
+    vulkano_shaders::shader! {
+        ty: "fragment",
+        src: r"#version 310 es
+            precision highp float;
+
+            layout (location = 0) in vec2 in_uv;
+            layout (location = 0) out vec4 out_color;
+
+            layout (set = 0, binding = 0) uniform ColorBlock {
+                uniform vec4 in_color;
+            };
+
+            void main()
+            {
+                out_color = in_color;
+            }
+        ",
+    }
+}
+
+pub mod frag_glyph {
+    vulkano_shaders::shader! {
+        ty: "fragment",
+        src: r"#version 310 es
+            precision highp float;
+
+            layout (location = 0) in vec2 in_uv;
+            layout (location = 0) out vec4 out_color;
+
+            layout (set = 0, binding = 0) uniform sampler2D in_texture;
+
+            layout (set = 1, binding = 0) uniform ColorBlock {
+                uniform vec4 in_color;
+            };
+
+            void main()
+            {
+                float r = texture(in_texture, in_uv).r;
+                out_color = vec4(r,r,r,r) * in_color;
+            }
+        ",
+    }
+}
+
+pub mod frag_sprite {
+    vulkano_shaders::shader! {
+        ty: "fragment",
+        src: r"#version 310 es
+            precision highp float;
+
+            layout (location = 0) in vec2 in_uv;
+            layout (location = 0) out vec4 out_color;
+
+            layout (set = 0, binding = 0) uniform sampler2D in_texture;
+
+            void main()
+            {
+                out_color = texture(in_texture, in_uv);
+            }
+        ",
+    }
+}
+
+pub mod frag_screen {
+    vulkano_shaders::shader! {
+        ty: "fragment",
+        src: r"#version 310 es
+            precision highp float;
+
+            layout (location = 0) in vec2 in_uv;
+            layout (location = 0) out vec4 out_color;
+
+            layout (set = 0, binding = 0) uniform sampler2D in_texture;
+
+            void main()
+            {
+                out_color = texture(in_texture, in_uv);
+
+                // linear to srgb
+                bvec4 cutoff = lessThan(out_color, vec4(0.0031308));
+                vec4 higher = (pow(out_color, vec4(0.41666)) * vec4(1.055)) - vec4(0.055);
+                vec4 lower = out_color*vec4(12.92);
+                out_color = mix(higher, lower, cutoff);
+                out_color.a = 1.0;
+            }
+        ",
+    }
+}
+
+pub mod frag_srgb {
+    vulkano_shaders::shader! {
+        ty: "fragment",
+        src: r"#version 310 es
+            precision highp float;
+
+            layout (location = 0) in vec2 in_uv;
+            layout (location = 0) out vec4 out_color;
+
+            layout (set = 0, binding = 0) uniform sampler2D in_texture;
+            layout (set = 1, binding = 0) uniform AlphaBlock {
+                uniform float alpha;
+            };
+
+
+            void main()
+            {
+                out_color = texture(in_texture, in_uv);
+                
+                bvec4 cutoff = lessThan(out_color, vec4(0.04045));
+                vec4 higher = pow((out_color + vec4(0.055))/vec4(1.055), vec4(2.4));
+                vec4 lower = out_color/vec4(12.92);
+
+                out_color = mix(higher, lower, cutoff);
+                out_color.a = alpha;
+            }
+        ",
+    }
+}

src/state.rs

@@ -0,0 +1,169 @@
+use std::{io::Cursor, path::PathBuf, sync::Arc};
+
+use anyhow::bail;
+use glam::Vec3;
+use rodio::{Decoder, OutputStream, OutputStreamHandle, Source};
+use serde::{Deserialize, Serialize};
+use smallvec::{smallvec, SmallVec};
+use vulkano::format::Format;
+
+use crate::{
+    backend::{common::TaskContainer, input::InputState},
+    config::GeneralConfig,
+    config_io,
+    graphics::WlxGraphics,
+    gui::font::FontCache,
+    hid::HidProvider,
+    shaders::{frag_color, frag_glyph, frag_screen, frag_sprite, frag_srgb, vert_common},
+};
+
+pub struct AppState {
+    pub fc: FontCache,
+    pub session: AppSession,
+    pub tasks: TaskContainer,
+    pub graphics: Arc<WlxGraphics>,
+    pub format: vulkano::format::Format,
+    pub input_state: InputState,
+    pub hid_provider: Box<dyn HidProvider>,
+    pub audio: AudioOutput,
+    pub screens: SmallVec<[ScreenMeta; 8]>,
+}
+
+impl AppState {
+    pub fn from_graphics(graphics: Arc<WlxGraphics>) -> anyhow::Result<Self> {
+        // insert shared resources
+        {
+            let Ok(mut shaders) = graphics.shared_shaders.write() else {
+                bail!("Failed to lock shared shaders");
+            };
+
+            let shader = vert_common::load(graphics.device.clone())?;
+            shaders.insert("vert_common", shader);
+
+            let shader = frag_color::load(graphics.device.clone())?;
+            shaders.insert("frag_color", shader);
+
+            let shader = frag_glyph::load(graphics.device.clone())?;
+            shaders.insert("frag_glyph", shader);
+
+            let shader = frag_sprite::load(graphics.device.clone())?;
+            shaders.insert("frag_sprite", shader);
+
+            let shader = frag_screen::load(graphics.device.clone())?;
+            shaders.insert("frag_screen", shader);
+
+            let shader = frag_srgb::load(graphics.device.clone())?;
+            shaders.insert("frag_srgb", shader);
+        }
+
+        Ok(AppState {
+            fc: FontCache::new()?,
+            session: AppSession::load(),
+            tasks: TaskContainer::new(),
+            graphics,
+            format: Format::R8G8B8A8_UNORM,
+            input_state: InputState::new(),
+            hid_provider: crate::hid::initialize(),
+            audio: AudioOutput::new(),
+            screens: smallvec![],
+        })
+    }
+}
+
+pub struct AppSession {
+    pub config_root_path: PathBuf,
+    pub config: GeneralConfig,
+
+    pub color_norm: Vec3,
+    pub color_shift: Vec3,
+    pub color_alt: Vec3,
+    pub color_grab: Vec3,
+}
+
+impl AppSession {
+    pub fn load() -> Self {
+        let config_root_path = config_io::ensure_config_root();
+        log::info!("Config root path: {}", config_root_path.to_string_lossy());
+        let config = GeneralConfig::load_from_disk();
+
+        AppSession {
+            config_root_path,
+            config,
+            color_norm: Vec3 {
+                x: 0.,
+                y: 1.,
+                z: 1.,
+            },
+            color_shift: Vec3 {
+                x: 1.,
+                y: 1.,
+                z: 0.,
+            },
+            color_alt: Vec3 {
+                x: 1.,
+                y: 0.,
+                z: 1.,
+            },
+            color_grab: Vec3 {
+                x: 1.,
+                y: 0.,
+                z: 0.,
+            },
+        }
+    }
+}
+
+pub struct AudioOutput {
+    audio_stream: Option<(OutputStream, OutputStreamHandle)>,
+    first_try: bool,
+}
+
+impl AudioOutput {
+    pub fn new() -> Self {
+        AudioOutput {
+            audio_stream: None,
+            first_try: true,
+        }
+    }
+
+    fn get_handle(&mut self) -> Option<&OutputStreamHandle> {
+        if self.audio_stream.is_none() && self.first_try {
+            self.first_try = false;
+            if let Ok((stream, handle)) = OutputStream::try_default() {
+                self.audio_stream = Some((stream, handle));
+            } else {
+                log::error!("Failed to open audio stream. Audio will not work.");
+                return None;
+            }
+        }
+        self.audio_stream.as_ref().map(|(_, h)| h)
+    }
+
+    pub fn play(&mut self, wav_bytes: &'static [u8]) {
+        let Some(handle) = self.get_handle() else {
+            return;
+        };
+        let cursor = Cursor::new(wav_bytes);
+        let source = match Decoder::new_wav(cursor) {
+            Ok(source) => source,
+            Err(e) => {
+                log::error!("Failed to play sound: {:?}", e);
+                return;
+            }
+        };
+        let _ = handle.play_raw(source.convert_samples());
+    }
+}
+
+pub struct ScreenMeta {
+    pub name: Arc<str>,
+    pub id: usize,
+}
+
+#[derive(Serialize, Deserialize, Clone, Copy, Default)]
+#[repr(u8)]
+pub enum LeftRight {
+    #[default]
+    Left,
+    Right,
+}

wlx-overlay-s.desktop

@@ -0,0 +1,7 @@
+[Desktop Entry]
+Type=Application
+Name=WlxOverlay-S
+Exec=wlx-overlay-s
+Icon=wlx-overlay-s
+Terminal=true
+Categories=Utility;