From 6ee78ec091a4516ce28d02d57f20c4f3f9b402b1 Mon Sep 17 00:00:00 2001 From: Julien Buret Date: Tue, 14 Apr 2026 20:21:09 +0200 Subject: [PATCH] feat(live): support live for gemini-3.1-flash-live-preview model --- .../adk/models/GeminiLlmConnection.java | 44 ++++++++- .../com/google/adk/utils/ModelNameUtils.java | 8 ++ .../adk/models/GeminiLlmConnectionTest.java | 89 +++++++++++++++++++ .../google/adk/utils/ModelNameUtilsTest.java | 15 ++++ 4 files changed, 155 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/com/google/adk/models/GeminiLlmConnection.java b/core/src/main/java/com/google/adk/models/GeminiLlmConnection.java index 1f3d0b8c5..21cb38523 100644 --- a/core/src/main/java/com/google/adk/models/GeminiLlmConnection.java +++ b/core/src/main/java/com/google/adk/models/GeminiLlmConnection.java @@ -18,6 +18,7 @@ import static com.google.common.collect.ImmutableList.toImmutableList; +import com.google.adk.utils.ModelNameUtils; import com.google.common.collect.ImmutableList; import com.google.genai.AsyncSession; import com.google.genai.Client; @@ -257,6 +258,13 @@ public Completable sendContent(Content content) { List functionResponses = extractFunctionResponses(content); if (functionResponses.isEmpty()) { + Optional realtimeInputParameters = + createRealtimeInputForContent(content, modelName, apiClient.vertexAI()); + if (realtimeInputParameters.isPresent()) { + return Completable.fromFuture( + sessionFuture.thenCompose( + session -> session.sendRealtimeInput(realtimeInputParameters.get()))); + } return sendClientContentInternal( LiveSendClientContentParameters.builder() .turns(ImmutableList.of(content)) @@ -289,7 +297,7 @@ public Completable sendRealtime(Blob blob) { sessionFuture.thenCompose( session -> session.sendRealtimeInput( - LiveSendRealtimeInputParameters.builder().media(blob).build()))); + createRealtimeInputForBlob(blob, modelName, apiClient.vertexAI())))); } /** Helper to send client content parameters. */ @@ -304,6 +312,40 @@ private Completable sendToolResponseInternal(LiveSendToolResponseParameters para sessionFuture.thenCompose(session -> session.sendToolResponse(parameters))); } + static boolean usesGemini31FlashLiveRealtimeInput(String modelName, boolean vertexAI) { + return !vertexAI && ModelNameUtils.isGemini31FlashLiveModel(modelName); + } + + static Optional createRealtimeInputForContent( + Content content, String modelName, boolean vertexAI) { + if (!usesGemini31FlashLiveRealtimeInput(modelName, vertexAI) + || content.parts().isEmpty() + || content.parts().get().size() != 1) { + return Optional.empty(); + } + Part part = content.parts().get().get(0); + if (part.text().isEmpty()) { + return Optional.empty(); + } + return Optional.of(LiveSendRealtimeInputParameters.builder().text(part.text().get()).build()); + } + + static LiveSendRealtimeInputParameters createRealtimeInputForBlob( + Blob blob, String modelName, boolean vertexAI) { + if (!usesGemini31FlashLiveRealtimeInput(modelName, vertexAI)) { + return LiveSendRealtimeInputParameters.builder().media(blob).build(); + } + + String mimeType = blob.mimeType().orElse(""); + if (mimeType.startsWith("audio/")) { + return LiveSendRealtimeInputParameters.builder().audio(blob).build(); + } + if (mimeType.startsWith("image/")) { + return LiveSendRealtimeInputParameters.builder().video(blob).build(); + } + return LiveSendRealtimeInputParameters.builder().media(blob).build(); + } + @Override public Flowable receive() { return responseFlowable; diff --git a/core/src/main/java/com/google/adk/utils/ModelNameUtils.java b/core/src/main/java/com/google/adk/utils/ModelNameUtils.java index cf0f2221e..3a873ef7e 100644 --- a/core/src/main/java/com/google/adk/utils/ModelNameUtils.java +++ b/core/src/main/java/com/google/adk/utils/ModelNameUtils.java @@ -24,6 +24,7 @@ /** Utility class for model names. */ public final class ModelNameUtils { private static final String GEMINI_PREFIX = "gemini-"; + private static final String GEMINI_3_1_FLASH_LIVE_PREFIX = "gemini-3.1-flash-live"; private static final Pattern GEMINI_2_PATTERN = Pattern.compile("^gemini-2\\..*"); private static final String GEMINI_CLASS = "com.google.adk.models.Gemini"; private static final Pattern PATH_PATTERN = @@ -39,6 +40,13 @@ public static boolean isGemini2Model(String modelString) { return matchesModelPattern(modelString, GEMINI_2_PATTERN); } + public static boolean isGemini31FlashLiveModel(String modelString) { + if (modelString == null) { + return false; + } + return extractModelName(modelString).startsWith(GEMINI_3_1_FLASH_LIVE_PREFIX); + } + private static boolean matchesModelPattern(String modelString, Pattern pattern) { if (modelString == null) { return false; diff --git a/core/src/test/java/com/google/adk/models/GeminiLlmConnectionTest.java b/core/src/test/java/com/google/adk/models/GeminiLlmConnectionTest.java index a3ac09fe5..8856a7185 100644 --- a/core/src/test/java/com/google/adk/models/GeminiLlmConnectionTest.java +++ b/core/src/test/java/com/google/adk/models/GeminiLlmConnectionTest.java @@ -19,9 +19,11 @@ import static com.google.common.truth.Truth.assertThat; import com.google.common.collect.ImmutableList; +import com.google.genai.types.Blob; import com.google.genai.types.Content; import com.google.genai.types.FunctionCall; import com.google.genai.types.GenerateContentResponseUsageMetadata; +import com.google.genai.types.LiveSendRealtimeInputParameters; import com.google.genai.types.LiveServerContent; import com.google.genai.types.LiveServerMessage; import com.google.genai.types.LiveServerSetupComplete; @@ -31,6 +33,7 @@ import com.google.genai.types.UsageMetadata; import io.reactivex.rxjava3.observers.TestObserver; import java.util.List; +import java.util.Optional; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -38,6 +41,92 @@ @RunWith(JUnit4.class) public final class GeminiLlmConnectionTest { + @Test + public void usesGemini31FlashLiveRealtimeInput_withGeminiApiPreviewModel_returnsTrue() { + assertThat( + GeminiLlmConnection.usesGemini31FlashLiveRealtimeInput( + "gemini-3.1-flash-live-preview", false)) + .isTrue(); + } + + @Test + public void usesGemini31FlashLiveRealtimeInput_withVertexAi_returnsFalse() { + assertThat( + GeminiLlmConnection.usesGemini31FlashLiveRealtimeInput( + "gemini-3.1-flash-live-preview", true)) + .isFalse(); + } + + @Test + public void createRealtimeInputForContent_withSingleTextOnGemini31_returnsTextInput() { + Content content = Content.fromParts(Part.fromText("hello")); + + Optional parameters = + GeminiLlmConnection.createRealtimeInputForContent( + content, "gemini-3.1-flash-live-preview", false); + + assertThat(parameters).isPresent(); + assertThat(parameters.get().text()).hasValue("hello"); + assertThat(parameters.get().media()).isEmpty(); + } + + @Test + public void createRealtimeInputForContent_withNonTextContent_returnsEmpty() { + Content content = + Content.builder() + .parts( + ImmutableList.of( + Part.builder() + .inlineData( + Blob.builder().mimeType("image/png").data(new byte[] {1}).build()) + .build())) + .build(); + + Optional parameters = + GeminiLlmConnection.createRealtimeInputForContent( + content, "gemini-3.1-flash-live-preview", false); + + assertThat(parameters).isEmpty(); + } + + @Test + public void createRealtimeInputForBlob_withGemini31Audio_setsAudio() { + Blob blob = Blob.builder().mimeType("audio/pcm").data(new byte[] {1}).build(); + + LiveSendRealtimeInputParameters parameters = + GeminiLlmConnection.createRealtimeInputForBlob( + blob, "gemini-3.1-flash-live-preview", false); + + assertThat(parameters.audio()).hasValue(blob); + assertThat(parameters.media()).isEmpty(); + assertThat(parameters.video()).isEmpty(); + } + + @Test + public void createRealtimeInputForBlob_withGemini31Image_setsVideo() { + Blob blob = Blob.builder().mimeType("image/jpeg").data(new byte[] {1}).build(); + + LiveSendRealtimeInputParameters parameters = + GeminiLlmConnection.createRealtimeInputForBlob( + blob, "gemini-3.1-flash-live-preview", false); + + assertThat(parameters.video()).hasValue(blob); + assertThat(parameters.media()).isEmpty(); + assertThat(parameters.audio()).isEmpty(); + } + + @Test + public void createRealtimeInputForBlob_withOtherModel_keepsMedia() { + Blob blob = Blob.builder().mimeType("audio/pcm").data(new byte[] {1}).build(); + + LiveSendRealtimeInputParameters parameters = + GeminiLlmConnection.createRealtimeInputForBlob(blob, "gemini-2.0-flash-live-001", false); + + assertThat(parameters.media()).hasValue(blob); + assertThat(parameters.audio()).isEmpty(); + assertThat(parameters.video()).isEmpty(); + } + @Test public void convertToServerResponse_withInterruptedTrue_mapsInterruptedField() { LiveServerContent serverContent = diff --git a/core/src/test/java/com/google/adk/utils/ModelNameUtilsTest.java b/core/src/test/java/com/google/adk/utils/ModelNameUtilsTest.java index 20dda7034..bbbee84d6 100644 --- a/core/src/test/java/com/google/adk/utils/ModelNameUtilsTest.java +++ b/core/src/test/java/com/google/adk/utils/ModelNameUtilsTest.java @@ -137,6 +137,21 @@ public void isGeminiModel_withEmptyModel_returnsFalse() { assertThat(ModelNameUtils.isGeminiModel("")).isFalse(); } + @Test + public void isGemini31FlashLiveModel_withPrefixOnly_returnsTrue() { + assertThat(ModelNameUtils.isGemini31FlashLiveModel("gemini-3.1-flash-live")).isTrue(); + } + + @Test + public void isGemini31FlashLiveModel_withPrefixedVariant_returnsTrue() { + assertThat(ModelNameUtils.isGemini31FlashLiveModel("gemini-3.1-flash-live-preview")).isTrue(); + } + + @Test + public void isGemini31FlashLiveModel_withOtherModel_returnsFalse() { + assertThat(ModelNameUtils.isGemini31FlashLiveModel("gemini-2.0-flash-live-001")).isFalse(); + } + @Test public void isInstanceOfGemini_withGeminiInstance_returnsTrue() { assertThat(ModelNameUtils.isInstanceOfGemini(new Gemini("", ""))).isTrue();