google ai support vision request

jcommon/ai/google/pom.xml

@@ -10,7 +10,7 @@
     </parent>
 
     <artifactId>google</artifactId>
-    <version>1.4-jdk8-SNAPSHOT</version>
+    <version>1.5-jdk8-SNAPSHOT</version>
 
     <properties>
         <maven.compiler.source>8</maven.compiler.source>

jcommon/ai/google/src/main/java/run/mone/ai/google/CloudeClient.java

@@ -12,6 +12,7 @@
 import run.mone.ai.google.bo.Content;
 import run.mone.ai.google.bo.RequestPayload;
 import run.mone.ai.google.bo.ResponsePayload;
+import run.mone.ai.google.bo.multiModal.GVisionRequest;
 
 import java.io.FileInputStream;
 import java.io.IOException;
@@ -38,7 +39,6 @@ public class CloudeClient {
 
     private static Gson gson = new Gson();
 
-
     @SneakyThrows
     public String token(String model) {
         GoogleCredentials credentials = GoogleCredentials.fromStream(
@@ -52,42 +52,23 @@ public String token(String model) {
         return this.token;
     }
 
+    public ResponsePayload visionCall(String url, String token, GVisionRequest GVisionRequest) {
+        return baseCall(url, token, gson.toJson(GVisionRequest));
+    }
 
     public ResponsePayload call(String token, RequestPayload requestPayload) {
-        OkHttpClient client = new OkHttpClient.Builder().readTimeout(5, TimeUnit.MINUTES).build();
-        MediaType mediaType = MediaType.parse("application/json; charset=utf-8");
-        RequestBody body = RequestBody.create(mediaType, new Gson().toJson(requestPayload));
-        Request request = new Request.Builder()
-                .url(url + projectId + "/locations/us-central1/publishers/anthropic/models/" + model + ":streamRawPredict")
-                .post(body)
-                .addHeader("Authorization", "Bearer " + token)
-                .addHeader("Content-Type", "application/json; charset=utf-8")
-                .build();
+        String callUrl = url + projectId + "/locations/us-central1/publishers/anthropic/models/" + model + ":streamRawPredict";
+        return call(callUrl, token, requestPayload);
+    }
 
-        try (Response response = client.newCall(request).execute()) {
-            if (response.code() == 429) {
-                ResponsePayload res = new ResponsePayload();
-                Content content = new Content();
-                content.setText(gson.toJson(ImmutableMap.of("message", "被claude3限流了", "code", "429")));
-                log.info("claude res:{}", content.getText());
-                res.setContent(Lists.newArrayList(content));
-                return res;
-            }
-            if (!response.isSuccessful()) throw new IOException("Unexpected code " + response);
-            // Handle the response
-            String res = response.body().string();
-            log.info("claude3 res:{}", res);
-            return new Gson().fromJson(res, ResponsePayload.class);
-        } catch (Throwable e) {
-            log.error(e.getMessage(), e);
-        }
-        return null;
+    public ResponsePayload call(String url, String token, RequestPayload requestPayload) {
+        return baseCall(url, token, gson.toJson(requestPayload));
     }
 
-    public ResponsePayload call(String url ,String token, RequestPayload requestPayload) {
+    private ResponsePayload baseCall(String url, String token, String bodyStr) {
         OkHttpClient client = new OkHttpClient.Builder().readTimeout(5, TimeUnit.MINUTES).build();
         MediaType mediaType = MediaType.parse("application/json; charset=utf-8");
-        RequestBody body = RequestBody.create(mediaType, new Gson().toJson(requestPayload));
+        RequestBody body = RequestBody.create(mediaType, bodyStr);
         Request request = new Request.Builder()
                 .url(url)
                 .post(body)

jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionContent.java

@@ -0,0 +1,19 @@
+package run.mone.ai.google.bo.multiModal;
+
+import com.google.gson.annotations.SerializedName;
+import lombok.Builder;
+import lombok.Data;
+
+@Data
+@Builder
+public class GVisionContent {
+
+    @SerializedName("type")
+    private String type;
+
+    @SerializedName("text")
+    private String text;
+
+    @SerializedName("source")
+    private GVisionSource source;
+}

jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionMsg.java

@@ -0,0 +1,23 @@
+package run.mone.ai.google.bo.multiModal;
+
+import com.google.gson.annotations.SerializedName;
+import lombok.Builder;
+import lombok.Data;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * @author [email protected]
+ * @date 2023/5/25 14:16
+ */
+@Data
+@Builder
+public class GVisionMsg implements Serializable {
+
+    @SerializedName("role")
+    private String role;
+
+    @SerializedName("content")
+    private List<GVisionContent> content;
+}

jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionRequest.java

@@ -0,0 +1,32 @@
+package run.mone.ai.google.bo.multiModal;
+
+import com.google.gson.annotations.SerializedName;
+import lombok.Builder;
+import lombok.Data;
+
+import java.util.List;
+
+/**
+ * @author [email protected]
+ * @date 2024/4/9 16:36
+ */
+@Data
+@Builder
+public class GVisionRequest {
+
+
+        @SerializedName("anthropic_version")
+        private String anthropicVersion;
+
+        @SerializedName("messages")
+        private List<GVisionMsg> messages;
+
+        @SerializedName("max_tokens")
+        private int maxTokens;
+
+        @SerializedName("stream")
+        private boolean stream;
+
+
+
+}

jcommon/ai/google/src/main/java/run/mone/ai/google/bo/multiModal/GVisionSource.java

@@ -0,0 +1,20 @@
+package run.mone.ai.google.bo.multiModal;
+
+import com.google.gson.annotations.SerializedName;
+import lombok.Builder;
+import lombok.Data;
+
+@Data
+@Builder
+public class GVisionSource {
+
+    @SerializedName("type")
+    private String type;
+
+    @SerializedName("media_type")
+    private String mediaType;
+
+    @SerializedName("data")
+    private String data;
+
+}