feat: privileged audio capture via REMOTE_SUBMIX

allows navigation instructions to be captured
This commit is contained in:
Benjamin Wiegand
2026-06-20 15:28:44 -07:00
parent b6974a05f2
commit 6a62f1b798
15 changed files with 471 additions and 14 deletions
@@ -13,6 +13,7 @@ import android.util.Log;
import javax.net.ssl.SSLException;
import io.benwiegand.projection.geargrinder.ConnectionService;
import io.benwiegand.projection.geargrinder.projection.PrivdAudioService;
import io.benwiegand.projection.geargrinder.projection.audio.LocalAudioRecordCapture;
import io.benwiegand.projection.geargrinder.crypto.TLSService;
import io.benwiegand.projection.geargrinder.message.MessageBroker;
@@ -38,6 +39,8 @@ public class ControlChannel implements MessageListener, ProjectionService.Listen
private static final int VERSION_CODE_MAJOR = 1;
private static final int VERSION_CODE_MINOR = 7;
private final Object lock = new Object();
private final Context context;
private final ConnectionService.ServiceBinder connectionServiceBinder;
private final MessageBroker mb;
@@ -49,6 +52,8 @@ public class ControlChannel implements MessageListener, ProjectionService.Listen
private final ControlListener controlListener;
private PrivdAudioService privdAudioService = null;
private ServiceDiscoveryResponse serviceDiscoveryResponse = null;
private VideoChannel videoChannel = null;
@@ -73,21 +78,45 @@ public class ControlChannel implements MessageListener, ProjectionService.Listen
if (mediaAudioChannel != null) mediaAudioChannel.destroy();
if (inputChannel != null) inputChannel.destroy();
if (sensorChannel != null) sensorChannel.destroy();
if (privdAudioService != null) privdAudioService.destroy();
}
private AudioChannel.AudioCaptureProvider createAudioCaptureProvider(AudioChannelMeta acm) {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
Log.e(TAG, "can't launch audio capture through MediaProjection: android version too old");
return null;
private void setupPrivdAudioService() {
synchronized (lock) {
if (privdAudioService != null) return;
Log.i(TAG, "setting up privd audio service");
privdAudioService = new PrivdAudioService(context);
}
}
LocalAudioRecordCapture.MediaProjectionProvider captureProvider = new LocalAudioRecordCapture.MediaProjectionProvider(context);
connectionServiceBinder.requestMediaProjection(mediaProjection -> {
if (!mb.isAlive()) return;
captureProvider.setMediaProjection(mediaProjection);
});
private AudioChannel.AudioCaptureProvider createMediaAudioCaptureProvider() {
return switch (settingsManager.getMediaAudioCaptureMode()) {
case DISABLED -> {
Log.w(TAG, "media audio capture is disabled");
yield null;
}
case MEDIA_PROJECTION -> {
Log.i(TAG, "using media projection for media audio capture");
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.Q) {
Log.e(TAG, "can't launch audio capture through MediaProjection: android version too old");
yield null;
}
return captureProvider;
LocalAudioRecordCapture.MediaProjectionProvider captureProvider = new LocalAudioRecordCapture.MediaProjectionProvider(context);
connectionServiceBinder.requestMediaProjection(mediaProjection -> {
if (!mb.isAlive()) return;
captureProvider.setMediaProjection(mediaProjection);
});
yield captureProvider;
}
case REMOTE_SUBMIX -> {
Log.i(TAG, "using REMOTE_SUBMIX for media audio capture");
setupPrivdAudioService();
yield privdAudioService.getMediaAudioCaptureProvider();
}
};
}
private void startProjection() {
@@ -122,9 +151,9 @@ public class ControlChannel implements MessageListener, ProjectionService.Listen
continue;
}
AudioChannel.AudioCaptureProvider audioCaptureProvider = createAudioCaptureProvider(acm);
AudioChannel.AudioCaptureProvider audioCaptureProvider = createMediaAudioCaptureProvider();
if (audioCaptureProvider == null) {
Log.e(TAG, "unable to create audio capture provider for media audio channel");
Log.w(TAG, "no audio capture provider for media audio channel");
Log.w(TAG, "not initializing: " + acm);
continue;
}
@@ -19,6 +19,7 @@ import java.util.List;
import io.benwiegand.projection.geargrinder.NotificationService;
import io.benwiegand.projection.geargrinder.R;
import io.benwiegand.projection.geargrinder.SettingsActivity;
import io.benwiegand.projection.geargrinder.settings.MediaAudioCaptureMode;
import io.benwiegand.projection.geargrinder.settings.PrivilegeMode;
import rikka.shizuku.Shizuku;
@@ -40,7 +41,8 @@ public class PermissionRequirements {
public static final PermissionEntry RECORD_AUDIO_PERMISSION_ENTRY = PermissionEntry.createForManifestPermission(
Manifest.permission.RECORD_AUDIO, Build.VERSION_CODES.Q, null, // doesn't seem to be needed on newer android versions, but the docs still say otherwise
R.string.record_audio_permission_title, R.string.record_audio_permission_rationale,
settingsManager -> true, settingsManager -> true);
settingsManager -> settingsManager.getMediaAudioCaptureMode() == MediaAudioCaptureMode.MEDIA_PROJECTION,
settingsManager -> true);
public static final PermissionEntry READ_PHONE_STATE_PERMISSION_ENTRY = PermissionEntry.createForManifestPermission(
Manifest.permission.READ_PHONE_STATE, Build.VERSION_CODES.Q, Build.VERSION_CODES.R,
@@ -0,0 +1,65 @@
package io.benwiegand.projection.geargrinder.projection;
import static android.content.Context.BIND_AUTO_CREATE;
import static android.content.Context.BIND_IMPORTANT;
import android.content.Context;
import android.media.MediaRecorder;
import android.util.Log;
import io.benwiegand.projection.geargrinder.PrivdService;
import io.benwiegand.projection.geargrinder.callback.IPCConnectionListener;
import io.benwiegand.projection.geargrinder.channel.AudioChannel;
import io.benwiegand.projection.geargrinder.projection.audio.PrivdAudioCaptureProxy;
import io.benwiegand.projection.geargrinder.service.GeargrinderServiceConnector;
import io.benwiegand.projection.libprivd.IPrivd;
public class PrivdAudioService implements GeargrinderServiceConnector.ConnectionListener, IPCConnectionListener {
private static final String TAG = PrivdAudioService.class.getSimpleName();
private final GeargrinderServiceConnector connector;
private final PrivdAudioCaptureProxy.PrivdProvider mediaAudioCaptureProvider = new PrivdAudioCaptureProxy.PrivdProvider(MediaRecorder.AudioSource.REMOTE_SUBMIX);
private boolean dead = false;
public PrivdAudioService(Context context) {
connector = new GeargrinderServiceConnector(TAG, context, this);
connector.bindPrivdService(BIND_AUTO_CREATE | BIND_IMPORTANT);
}
public void destroy() {
if (dead) return;
dead = true;
connector.destroy();
}
public AudioChannel.AudioCaptureProvider getMediaAudioCaptureProvider() {
return mediaAudioCaptureProvider;
}
@Override
public void onPrivdServiceConnected(PrivdService.ServiceBinder binder) {
binder.requestDaemon(this);
}
@Override
public void onPrivdConnected(IPrivd privd) {
if (dead) return;
Log.i(TAG, "privd connected");
mediaAudioCaptureProvider.setPrivd(privd);
}
@Override
public void onPrivdDisconnected() {
// TODO
Log.e(TAG, "privd connection lost");
}
@Override
public void onPrivdLaunchFailure(Throwable t) {
// TODO
Log.e(TAG, "privd failed to launch", t);
}
}
@@ -0,0 +1,103 @@
package io.benwiegand.projection.geargrinder.projection.audio;
import android.os.RemoteException;
import android.util.Log;
import io.benwiegand.projection.geargrinder.channel.AudioChannel;
import io.benwiegand.projection.geargrinder.proto.data.readable.av.preset.AudioPreset;
import io.benwiegand.projection.libprivd.IPrivd;
import io.benwiegand.projection.libprivd.audio.AudioCapture;
import io.benwiegand.projection.libprivd.audio.AudioCaptureResult;
public class PrivdAudioCaptureProxy implements AudioCapture {
private static final String TAG = PrivdAudioCaptureProxy.class.getSimpleName();
private final IPrivd privd;
private final int id;
public PrivdAudioCaptureProxy(IPrivd privd, AudioPreset preset, int bufferSize, int audioSource) throws RemoteException {
this.privd = privd;
id = privd.createPrivilegedAudioRecordCapture(preset.createAudioFormat(), bufferSize, audioSource);
}
@Override
public void begin() {
try {
privd.audioCaptureBegin(id);
} catch (RemoteException e) {
throw new RuntimeException("failed to begin audio capture", e);
}
}
@Override
public void destroy() {
try {
privd.destroyAudioCapture(id);
} catch (RemoteException e) {
Log.w(TAG, "failed to destroy audio capture", e);
// don't throw, privd is probably dead and the audio capture is destroyed anyway
}
}
@Override
public void nextBuffer(AudioCaptureResult result, byte[] buffer, int offset, int length) {
try {
privd.audioCaptureNextBuffer(id, result, buffer, offset, length);
} catch (RemoteException e) {
throw new RuntimeException("failed to get next buffer for audio capture", e);
}
}
public static class PrivdProvider implements AudioChannel.AudioCaptureProvider {
private static final String TAG = LocalAudioRecordCapture.MediaProjectionProvider.class.getSimpleName();
private final Object lock = new Object();
private final int audioSource;
private Runnable onReady = null;
private IPrivd privd = null;
public PrivdProvider(int audioSource) {
this.audioSource = audioSource;
}
public void setPrivd(IPrivd privd) {
synchronized (lock) {
this.privd = privd;
if (onReady != null) {
onReady.run();
onReady = null;
}
}
}
@Override
public void registerReadyCallback(Runnable onReady) {
synchronized (lock) {
if (privd != null) {
onReady.run();
return;
}
this.onReady = onReady;
}
}
@Override
public AudioCapture getInstance(AudioPreset audioPreset, int bufferSize) {
if (privd == null) {
Log.wtf(TAG, "getInstance() called before onReady!!!");
assert false;
return null;
}
try {
return new PrivdAudioCaptureProxy(privd, audioPreset, bufferSize, audioSource);
} catch (Throwable t) {
Log.e(TAG, "failed to initialize privd audio capture proxy", t);
return null;
}
}
}
}
@@ -0,0 +1,27 @@
package io.benwiegand.projection.geargrinder.settings;
import android.content.Context;
import android.util.Pair;
import java.util.List;
import io.benwiegand.projection.geargrinder.R;
public enum MediaAudioCaptureMode {
DISABLED,
MEDIA_PROJECTION,
REMOTE_SUBMIX;
public static MediaAudioCaptureMode parse(Context context, String value) {
return SettingsManager.enumForPref(
context, value,
R.string.key_media_audio_capture_mode,
R.string.media_audio_capture_mode_default,
List.of(
Pair.create(R.string.media_audio_capture_mode_disabled, DISABLED),
Pair.create(R.string.media_audio_capture_mode_media_projection, MEDIA_PROJECTION),
Pair.create(R.string.media_audio_capture_mode_remote_submix, REMOTE_SUBMIX)
)
);
}
}
@@ -53,6 +53,10 @@ public class SettingsManager {
return getBool(R.string.key_video_frame_rate_optimization, R.string.video_frame_rate_optimization_default);
}
public MediaAudioCaptureMode getMediaAudioCaptureMode() {
return MediaAudioCaptureMode.parse(context, getString(R.string.key_media_audio_capture_mode));
}
public boolean useImportedPhoneKeys() {
return getBool(R.string.key_use_imported_phone_keys, R.string.start_projection_when_locked_default);
}
+22
View File
@@ -29,6 +29,12 @@
<item>Constant quality</item>
</string-array>
<string-array name="media_audio_capture_mode_entries">
<item>disabled</item>
<item>screen sharing (limited, requires Android 10 or later)</item>
<item>remote submix (requires root or Shizuku)</item>
</string-array>
<string translatable="false" name="privilege_mode_no_root">no_root</string>
<string translatable="false" name="privilege_mode_shizuku">shizuku</string>
@@ -88,9 +94,23 @@
<string translatable="false" name="video_bitrate_custom_default">-1</string>
<string translatable="false" name="video_frame_rate_optimization_default">true</string>
<string translatable="false" name="media_audio_capture_mode_disabled">disabled</string>
<string translatable="false" name="media_audio_capture_mode_media_projection">media_projection</string>
<string translatable="false" name="media_audio_capture_mode_remote_submix">remote_submix</string>
<string translatable="false" name="media_audio_capture_mode_default">@string/media_audio_capture_mode_remote_submix</string>
<string-array translatable="false" name="media_audio_capture_mode_values">
<item>@string/media_audio_capture_mode_disabled</item>
<item>@string/media_audio_capture_mode_media_projection</item>
<item>@string/media_audio_capture_mode_remote_submix</item>
</string-array>
<string translatable="false" name="start_projection_when_locked_default">false</string>
@@ -104,6 +124,8 @@
<string translatable="false" name="key_video_bitrate_custom">video_bitrate_custom</string>
<string translatable="false" name="key_video_frame_rate_optimization">video_frame_rate_optimization</string>
<string translatable="false" name="key_media_audio_capture_mode">media_audio_capture_mode</string>
<string translatable="false" name="key_imported_phone_x509_certificate_chain">imported_phone_x509_certificate_chain</string>
<string translatable="false" name="key_imported_phone_pkcs8_private_key">imported_phone_pkcs8_private_key</string>
<string translatable="false" name="key_self_signed_phone_x509_certificate_chain">self_signed_phone_x509_certificate_chain</string>
+3
View File
@@ -172,4 +172,7 @@
<string name="bluetooth_connection_error_missing_permission">Missing Bluetooth permission.</string>
<string name="bluetooth_connection_error_general_failure">Failed to establish Bluetooth connection.</string>
<string name="pref_category_audio">audio</string>
<string name="media_audio_capture_mode_name">Media audio capture mode</string>
</resources>
@@ -105,4 +105,21 @@
</PreferenceCategory>
<PreferenceCategory
app:title="@string/pref_category_audio"
app:iconSpaceReserved="false"
>
<ListPreference
app:key="@string/key_media_audio_capture_mode"
app:iconSpaceReserved="false"
app:title="@string/media_audio_capture_mode_name"
app:entries="@array/media_audio_capture_mode_entries"
app:entryValues="@array/media_audio_capture_mode_values"
app:defaultValue="@string/media_audio_capture_mode_default"
app:useSimpleSummaryProvider="true"
/>
</PreferenceCategory>
</PreferenceScreen>
@@ -1,5 +1,9 @@
package io.benwiegand.projection.libprivd;
import io.benwiegand.projection.libprivd.audio.AudioCaptureResult;
parcelable AudioCaptureResult;
interface IPrivd {
void ping();
@@ -16,4 +20,12 @@ interface IPrivd {
void virtualDisplayResize(int displayId, int width, int height, int densityDpi);
void virtualDisplaySetSurface(int displayId, in Surface surface);
int createPrivilegedAudioRecordCapture(in AudioFormat audioFormat, int bufferSize, int audioSource);
void destroyAudioCapture(int id);
void audioCaptureBegin(int id);
void audioCaptureNextBuffer(int id, out AudioCaptureResult result, out byte[] buffer, int offset, int length);
}
@@ -0,0 +1,4 @@
package io.benwiegand.projection.libprivd.audio;
parcelable AudioCaptureResult;
@@ -5,6 +5,7 @@ import static io.benwiegand.projection.libprivd.ipc.IPCConstants.APP_PKG_NAME;
import static io.benwiegand.projection.libprivd.ipc.IPCConstants.BIND_TIMEOUT;
import static io.benwiegand.projection.libprivd.ipc.IPCConstants.PING_TIMEOUT;
import android.annotation.SuppressLint;
import android.app.ActivityOptions;
import android.content.ComponentName;
import android.content.Context;
@@ -12,6 +13,7 @@ import android.content.Intent;
import android.hardware.display.DisplayManager;
import android.hardware.display.VirtualDisplay;
import android.hardware.input.InputManager;
import android.media.AudioFormat;
import android.os.Binder;
import android.os.Build;
import android.os.Handler;
@@ -27,19 +29,26 @@ import java.io.IOException;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicInteger;
import io.benwiegand.projection.geargrinder.privd.audio.PrivilegedAudioRecordCapture;
import io.benwiegand.projection.geargrinder.privd.reflected.ReflectedIActivityManager;
import io.benwiegand.projection.geargrinder.privd.reflected.ReflectedInputEvent;
import io.benwiegand.projection.geargrinder.privd.reflected.ReflectedInputManager;
import io.benwiegand.projection.geargrinder.privd.reflection.ReflectionException;
import io.benwiegand.projection.libprivd.IPrivd;
import io.benwiegand.projection.libprivd.audio.AudioCapture;
import io.benwiegand.projection.libprivd.audio.AudioCaptureResult;
public class Privd extends IPrivd.Stub {
private static final String TAG = Privd.class.getSimpleName();
private static final boolean LOG_DEBUG = true;
private static final boolean LOG_DEBUG = false;
private final Context context;
private final Handler handler = new Handler(Looper.getMainLooper());
private final Map<Integer, VirtualDisplay> virtualDisplays = new ConcurrentHashMap<>();
private final Map<Integer, AudioCapture> audioCaptures = new ConcurrentHashMap<>();
private final AtomicInteger audioCaptureIdCounter = new AtomicInteger(0);
private final int appUid;
private final DisplayManager dm;
@@ -48,7 +57,9 @@ public class Privd extends IPrivd.Stub {
private long lastPingAt = 0;
@SuppressLint("NotificationPermission")
public Privd(Context context, int appUid) {
this.context = context;
this.appUid = appUid;
DisplayManager dm;
@@ -175,6 +186,12 @@ public class Privd extends IPrivd.Stub {
return virtualDisplay;
}
private AudioCapture getAudioCapture(int id) {
AudioCapture audioCapture = audioCaptures.get(id);
if (audioCapture == null) throw new NoSuchElementException("no AudioCapture exists with id " + id);
return audioCapture;
}
@Override
public int createVirtualDisplay(String name, int width, int height, int densityDpi, Surface surface, int flags) {
Log.v(TAG, "creating virtual display: " + name);
@@ -216,4 +233,47 @@ public class Privd extends IPrivd.Stub {
VirtualDisplay virtualDisplay = getVirtualDisplay(displayId);
virtualDisplay.setSurface(surface);
}
@Override
public int createPrivilegedAudioRecordCapture(AudioFormat audioFormat, int bufferSize, int audioSource) throws RemoteException {
int id = audioCaptureIdCounter.getAndIncrement();
Log.v(TAG, "creating privileged audio record capture: " + id);
try {
// TODO: need foreground context on A11
PrivilegedAudioRecordCapture audioCapture = new PrivilegedAudioRecordCapture(context, audioFormat, bufferSize, audioSource);
audioCaptures.put(id, audioCapture);
} catch (Throwable t) {
Log.e(TAG, "failed to create privileged audio record capture", t);
throw new RemoteException("failed to create privileged audio record capture: " + t);
}
return id;
}
@Override
public void destroyAudioCapture(int id) {
Log.v(TAG, "destroying audio capture" + id);
AudioCapture audioCapture = audioCaptures.remove(id);
if (audioCapture == null) {
Log.w(TAG, "can't destroy audio capture " + id + " because it doesn't exist or was already destroyed");
return;
}
audioCapture.destroy();
}
@Override
public void audioCaptureBegin(int id) {
Log.v(TAG, "audio capture begin: " + id);
getAudioCapture(id).begin();
}
@Override
public void audioCaptureNextBuffer(int id, AudioCaptureResult result, byte[] buffer, int offset, int length) {
if (LOG_DEBUG) Log.d(TAG, "getting next buffer for audio capture: " + id);
getAudioCapture(id).nextBuffer(result, buffer, offset, length);
}
}
@@ -0,0 +1,51 @@
package io.benwiegand.projection.geargrinder.privd.audio;
import android.annotation.SuppressLint;
import android.content.Context;
import android.media.AudioAttributes;
import android.media.AudioFormat;
import android.media.AudioRecord;
import android.os.Build;
import android.util.Log;
import io.benwiegand.projection.geargrinder.privd.reflected.ReflectedAudioAttributesBuilder;
import io.benwiegand.projection.geargrinder.privd.reflected.ReflectedAudioRecordBuilder;
import io.benwiegand.projection.libprivd.audio.AudioRecordCapture;
public class PrivilegedAudioRecordCapture extends AudioRecordCapture {
private static final String TAG = PrivilegedAudioRecordCapture.class.getSimpleName();
@SuppressLint("MissingPermission")
private static AudioRecord createAudioRecord(Context context, AudioFormat audioFormat, int bufferSize, int audioSource) {
AudioRecord.Builder audioRecordBuilder = new AudioRecord.Builder()
.setAudioFormat(audioFormat)
.setBufferSizeInBytes(bufferSize)
.setAudioSource(audioSource);
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S)
audioRecordBuilder.setContext(context);
try {
AudioAttributes.Builder audioAttributesBuilder = new AudioAttributes.Builder();
new ReflectedAudioAttributesBuilder(audioAttributesBuilder)
.setInternalCapturePreset(audioSource)
.addTag(ReflectedAudioRecordBuilder.SUBMIX_FIXED_VOLUME);
new ReflectedAudioRecordBuilder(audioRecordBuilder)
.setAudioAttributes(audioAttributesBuilder.build());
} catch (Throwable t) {
Log.e(TAG, "failed to set custom audio attributes", t);
}
return audioRecordBuilder.build();
}
@SuppressLint("MissingPermission")
public PrivilegedAudioRecordCapture(Context context, AudioFormat audioFormat, int bufferSize, int audioSource) {
super(createAudioRecord(context, audioFormat, bufferSize, audioSource));
}
}
@@ -0,0 +1,31 @@
package io.benwiegand.projection.geargrinder.privd.reflected;
import android.media.AudioAttributes;
import java.lang.reflect.Method;
import io.benwiegand.projection.geargrinder.privd.reflection.ReflectedObject;
import io.benwiegand.projection.geargrinder.privd.reflection.ReflectionException;
public class ReflectedAudioAttributesBuilder extends ReflectedObject {
private final Method setInternalCapturePreset;
private final Method addTag;
public ReflectedAudioAttributesBuilder(AudioAttributes.Builder instance) {
super(instance, AudioAttributes.Builder.class);
setInternalCapturePreset = findMethod("setInternalCapturePreset", int.class);
addTag = findMethod("addTag", String.class);
}
public ReflectedAudioAttributesBuilder setInternalCapturePreset(int preset) throws ReflectionException {
invokeMethodNoException(setInternalCapturePreset, preset);
return this;
}
public ReflectedAudioAttributesBuilder addTag(String tag) throws ReflectionException {
invokeMethodNoException(addTag, tag);
return this;
}
}
@@ -0,0 +1,27 @@
package io.benwiegand.projection.geargrinder.privd.reflected;
import android.media.AudioAttributes;
import android.media.AudioRecord;
import java.lang.reflect.Method;
import io.benwiegand.projection.geargrinder.privd.reflection.ReflectedObject;
import io.benwiegand.projection.geargrinder.privd.reflection.ReflectionException;
public class ReflectedAudioRecordBuilder extends ReflectedObject {
public static final String SUBMIX_FIXED_VOLUME = "fixedVolume";
private final Method setAudioAttributes;
public ReflectedAudioRecordBuilder(AudioRecord.Builder instance) {
super(instance, AudioRecord.Builder.class);
setAudioAttributes = findMethod("setAudioAttributes", AudioAttributes.class);
}
public ReflectedAudioRecordBuilder setAudioAttributes(AudioAttributes attributes) throws ReflectionException {
invokeMethodNoException(setAudioAttributes, attributes);
return this;
}
}