diff --git a/src/main/java/io/github/ollama4j/OllamaAPI.java b/src/main/java/io/github/ollama4j/OllamaAPI.java index a399adf..90c747f 100644 --- a/src/main/java/io/github/ollama4j/OllamaAPI.java +++ b/src/main/java/io/github/ollama4j/OllamaAPI.java @@ -1091,7 +1091,7 @@ public class OllamaAPI { * @return an {@link OllamaAsyncResultStreamer} handle for polling and * retrieving streamed results */ - public OllamaAsyncResultStreamer generateAsync(String model, String prompt, boolean raw, boolean think) { + public OllamaAsyncResultStreamer generate(String model, String prompt, boolean raw, boolean think) { OllamaGenerateRequest ollamaRequestModel = new OllamaGenerateRequest(model, prompt); ollamaRequestModel.setRaw(raw); ollamaRequestModel.setThink(think); @@ -1103,147 +1103,57 @@ public class OllamaAPI { } /** - * With one or more image files, ask a question to a model running on Ollama - * server. This is a - * sync/blocking call. - * - * @param model the ollama model to ask the question to - * @param prompt the prompt/question text - * @param imageFiles the list of image files to use for the question - * @param options the Options object - More - * details on the options - * @param streamHandler optional callback consumer that will be applied every - * time a streamed response is received. If not set, the - * stream parameter of the request is set to false. - * @return OllamaResult that includes response text and time taken for response - * @throws OllamaBaseException if the response indicates an error status - * @throws IOException if an I/O error occurs during the HTTP request - * @throws InterruptedException if the operation is interrupted - */ - public OllamaResult generateWithImageFiles(String model, String prompt, List imageFiles, Options options, - OllamaStreamHandler streamHandler) throws OllamaBaseException, IOException, InterruptedException { - List images = new ArrayList<>(); - for (File imageFile : imageFiles) { - images.add(encodeFileToBase64(imageFile)); - } - OllamaGenerateRequest ollamaRequestModel = new OllamaGenerateRequest(model, prompt, images); - ollamaRequestModel.setOptions(options.getOptionsMap()); - return generateSyncForOllamaRequestModel(ollamaRequestModel, null, streamHandler); - } - - /** - * Convenience method to call Ollama API without streaming responses. + * Generates a response from a model running on the Ollama server using one or more images as input. *

- * Uses - * {@link #generateWithImageFiles(String, String, List, Options, OllamaStreamHandler)} + * This method allows you to provide images (as {@link File}, {@code byte[]}, or image URL {@link String}) + * along with a prompt to the specified model. The images are automatically encoded as base64 before being sent. + * Additional model options can be specified via the {@link Options} parameter. + *

* - * @throws OllamaBaseException if the response indicates an error status - * @throws IOException if an I/O error occurs during the HTTP request - * @throws InterruptedException if the operation is interrupted - */ - public OllamaResult generateWithImageFiles(String model, String prompt, List imageFiles, Options options) - throws OllamaBaseException, IOException, InterruptedException { - return generateWithImageFiles(model, prompt, imageFiles, options, null); - } - - /** - * With one or more image URLs, ask a question to a model running on Ollama - * server. This is a - * sync/blocking call. - * - * @param model the ollama model to ask the question to - * @param prompt the prompt/question text - * @param imageURLs the list of image URLs to use for the question - * @param options the Options object - More - * details on the options - * @param streamHandler optional callback consumer that will be applied every - * time a streamed response is received. If not set, the - * stream parameter of the request is set to false. - * @return OllamaResult that includes response text and time taken for response - * @throws OllamaBaseException if the response indicates an error status - * @throws IOException if an I/O error occurs during the HTTP request - * @throws InterruptedException if the operation is interrupted - * @throws URISyntaxException if the URI for the request is malformed - */ - public OllamaResult generateWithImageURLs(String model, String prompt, List imageURLs, Options options, - OllamaStreamHandler streamHandler) - throws OllamaBaseException, IOException, InterruptedException, URISyntaxException { - List images = new ArrayList<>(); - for (String imageURL : imageURLs) { - images.add(encodeByteArrayToBase64(Utils.loadImageBytesFromUrl(imageURL))); - } - OllamaGenerateRequest ollamaRequestModel = new OllamaGenerateRequest(model, prompt, images); - ollamaRequestModel.setOptions(options.getOptionsMap()); - return generateSyncForOllamaRequestModel(ollamaRequestModel, null, streamHandler); - } - - /** - * Convenience method to call Ollama API without streaming responses. *

- * Uses - * {@link #generateWithImageURLs(String, String, List, Options, OllamaStreamHandler)} + * If a {@code streamHandler} is provided, the response will be streamed and the handler will be called + * for each streamed response chunk. If {@code streamHandler} is {@code null}, streaming is disabled and + * the full response is returned synchronously. + *

* - * @throws OllamaBaseException if the response indicates an error status - * @throws IOException if an I/O error occurs during the HTTP request - * @throws InterruptedException if the operation is interrupted - * @throws URISyntaxException if the URI for the request is malformed - */ - public OllamaResult generateWithImageURLs(String model, String prompt, List imageURLs, Options options) - throws OllamaBaseException, IOException, InterruptedException, URISyntaxException { - return generateWithImageURLs(model, prompt, imageURLs, options, null); - } - - /** - * Synchronously generates a response using a list of image byte arrays. - *

- * This method encodes the provided byte arrays into Base64 and sends them to - * the Ollama server. - * - * @param model the Ollama model to use for generating the response + * @param model the name of the Ollama model to use for generating the response * @param prompt the prompt or question text to send to the model - * @param images the list of image data as byte arrays - * @param options the Options object - More - * details on the options - * @param streamHandler optional callback that will be invoked with each - * streamed response; if null, streaming is disabled - * @return OllamaResult containing the response text and the time taken for the - * response - * @throws OllamaBaseException if the response indicates an error status + * @param images a list of images to use for the question; each element must be a {@link File}, {@code byte[]}, or a URL {@link String} + * @param options the {@link Options} object containing model parameters; + * see Ollama model options documentation + * @param streamHandler an optional callback that is invoked for each streamed response chunk; + * if {@code null}, disables streaming and returns the full response synchronously + * @return an {@link OllamaResult} containing the response text and time taken for the response + * @throws OllamaBaseException if the response indicates an error status or an invalid image type is provided * @throws IOException if an I/O error occurs during the HTTP request * @throws InterruptedException if the operation is interrupted + * @throws URISyntaxException if an image URL is malformed */ - public OllamaResult generateWithImages(String model, String prompt, List images, Options options, - OllamaStreamHandler streamHandler) throws OllamaBaseException, IOException, InterruptedException { + public OllamaResult generateWithImages(String model, String prompt, List images, Options options, Map format, + OllamaStreamHandler streamHandler) throws OllamaBaseException, IOException, InterruptedException, URISyntaxException { List encodedImages = new ArrayList<>(); - for (byte[] image : images) { - encodedImages.add(encodeByteArrayToBase64(image)); + for (Object image : images) { + if (image instanceof File) { + LOG.debug("Using image file: {}", ((File) image).getAbsolutePath()); + encodedImages.add(encodeFileToBase64((File) image)); + } else if (image instanceof byte[]) { + LOG.debug("Using image bytes: {}", ((byte[]) image).length + " bytes"); + encodedImages.add(encodeByteArrayToBase64((byte[]) image)); + } else if (image instanceof String) { + LOG.debug("Using image URL: {}", image); + encodedImages.add(encodeByteArrayToBase64(Utils.loadImageBytesFromUrl((String) image))); + } else { + throw new OllamaBaseException("Unsupported image type. Please provide a File, byte[], or a URL String."); + } } OllamaGenerateRequest ollamaRequestModel = new OllamaGenerateRequest(model, prompt, encodedImages); + if (format != null) { + ollamaRequestModel.setFormat(format); + } ollamaRequestModel.setOptions(options.getOptionsMap()); return generateSyncForOllamaRequestModel(ollamaRequestModel, null, streamHandler); } - /** - * Convenience method to call the Ollama API using image byte arrays without - * streaming responses. - *

- * Uses - * {@link #generateWithImages(String, String, List, Options, OllamaStreamHandler)} - * - * @throws OllamaBaseException if the response indicates an error status - * @throws IOException if an I/O error occurs during the HTTP request - * @throws InterruptedException if the operation is interrupted - */ - public OllamaResult generateWithImages(String model, String prompt, List images, Options options) - throws OllamaBaseException, IOException, InterruptedException { - return generateWithImages(model, prompt, images, options, null); - } - /** * Ask a question to a model based on a given message stack (i.e. a chat * history). Creates a synchronous call to the api diff --git a/src/main/java/io/github/ollama4j/models/chat/OllamaChatRequestBuilder.java b/src/main/java/io/github/ollama4j/models/chat/OllamaChatRequestBuilder.java index 4a9caf9..38ff63a 100644 --- a/src/main/java/io/github/ollama4j/models/chat/OllamaChatRequestBuilder.java +++ b/src/main/java/io/github/ollama4j/models/chat/OllamaChatRequestBuilder.java @@ -95,7 +95,7 @@ public class OllamaChatRequestBuilder { } public OllamaChatRequestBuilder withGetJsonResponse() { - this.request.setReturnFormatJson(true); + this.request.setFormat("json"); return this; } diff --git a/src/main/java/io/github/ollama4j/models/generate/OllamaGenerateRequest.java b/src/main/java/io/github/ollama4j/models/generate/OllamaGenerateRequest.java index 3763f0a..6228327 100644 --- a/src/main/java/io/github/ollama4j/models/generate/OllamaGenerateRequest.java +++ b/src/main/java/io/github/ollama4j/models/generate/OllamaGenerateRequest.java @@ -7,40 +7,38 @@ import lombok.Getter; import lombok.Setter; import java.util.List; +import java.util.Map; @Getter @Setter -public class OllamaGenerateRequest extends OllamaCommonRequest implements OllamaRequestBody{ +public class OllamaGenerateRequest extends OllamaCommonRequest implements OllamaRequestBody { - private String prompt; - private List images; + private String prompt; + private List images; + private String system; + private String context; + private boolean raw; + private boolean think; - private String system; - private String context; - private boolean raw; - private boolean think; - - public OllamaGenerateRequest() { - } - - public OllamaGenerateRequest(String model, String prompt) { - this.model = model; - this.prompt = prompt; - } - - public OllamaGenerateRequest(String model, String prompt, List images) { - this.model = model; - this.prompt = prompt; - this.images = images; - } - - @Override - public boolean equals(Object o) { - if (!(o instanceof OllamaGenerateRequest)) { - return false; + public OllamaGenerateRequest() { } - return this.toString().equals(o.toString()); - } + public OllamaGenerateRequest(String model, String prompt) { + this.model = model; + this.prompt = prompt; + } + public OllamaGenerateRequest(String model, String prompt, List images) { + this.model = model; + this.prompt = prompt; + this.images = images; + } + + @Override + public boolean equals(Object o) { + if (!(o instanceof OllamaGenerateRequest)) { + return false; + } + return this.toString().equals(o.toString()); + } } diff --git a/src/main/java/io/github/ollama4j/models/generate/OllamaGenerateRequestBuilder.java b/src/main/java/io/github/ollama4j/models/generate/OllamaGenerateRequestBuilder.java index 713c46e..5afbcf3 100644 --- a/src/main/java/io/github/ollama4j/models/generate/OllamaGenerateRequestBuilder.java +++ b/src/main/java/io/github/ollama4j/models/generate/OllamaGenerateRequestBuilder.java @@ -28,7 +28,7 @@ public class OllamaGenerateRequestBuilder { } public OllamaGenerateRequestBuilder withGetJsonResponse(){ - this.request.setReturnFormatJson(true); + this.request.setFormat("json"); return this; } diff --git a/src/main/java/io/github/ollama4j/models/request/OllamaCommonRequest.java b/src/main/java/io/github/ollama4j/models/request/OllamaCommonRequest.java index 879d801..9057d41 100644 --- a/src/main/java/io/github/ollama4j/models/request/OllamaCommonRequest.java +++ b/src/main/java/io/github/ollama4j/models/request/OllamaCommonRequest.java @@ -15,9 +15,10 @@ import java.util.Map; public abstract class OllamaCommonRequest { protected String model; - @JsonSerialize(using = BooleanToJsonFormatFlagSerializer.class) - @JsonProperty(value = "format") - protected Boolean returnFormatJson; +// @JsonSerialize(using = BooleanToJsonFormatFlagSerializer.class) +// this can either be set to format=json or format={"key1": "val1", "key2": "val2"} + @JsonProperty(value = "format", required = false, defaultValue = "json") + protected Object format; protected Map options; protected String template; protected boolean stream; diff --git a/src/main/java/io/github/ollama4j/utils/BooleanToJsonFormatFlagSerializer.java b/src/main/java/io/github/ollama4j/utils/BooleanToJsonFormatFlagSerializer.java index 590b59e..ed7bf20 100644 --- a/src/main/java/io/github/ollama4j/utils/BooleanToJsonFormatFlagSerializer.java +++ b/src/main/java/io/github/ollama4j/utils/BooleanToJsonFormatFlagSerializer.java @@ -6,16 +6,15 @@ import com.fasterxml.jackson.databind.SerializerProvider; import java.io.IOException; -public class BooleanToJsonFormatFlagSerializer extends JsonSerializer{ +public class BooleanToJsonFormatFlagSerializer extends JsonSerializer { @Override public void serialize(Boolean value, JsonGenerator gen, SerializerProvider serializers) throws IOException { - gen.writeString("json"); + gen.writeString("json"); } @Override - public boolean isEmpty(SerializerProvider provider,Boolean value){ + public boolean isEmpty(SerializerProvider provider, Boolean value) { return !value; } - } diff --git a/src/test/java/io/github/ollama4j/integrationtests/OllamaAPIIntegrationTest.java b/src/test/java/io/github/ollama4j/integrationtests/OllamaAPIIntegrationTest.java index 51d8edf..6483199 100644 --- a/src/test/java/io/github/ollama4j/integrationtests/OllamaAPIIntegrationTest.java +++ b/src/test/java/io/github/ollama4j/integrationtests/OllamaAPIIntegrationTest.java @@ -602,9 +602,9 @@ class OllamaAPIIntegrationTest { throws OllamaBaseException, IOException, URISyntaxException, InterruptedException { api.pullModel(VISION_MODEL); - OllamaResult result = api.generateWithImageURLs(VISION_MODEL, "What is in this image?", + OllamaResult result = api.generateWithImages(VISION_MODEL, "What is in this image?", List.of("https://i.pinimg.com/736x/f9/4e/cb/f94ecba040696a3a20b484d2e15159ec.jpg"), - new OptionsBuilder().build()); + new OptionsBuilder().build(), null, null); assertNotNull(result); assertNotNull(result.getResponse()); assertFalse(result.getResponse().isEmpty()); @@ -617,8 +617,8 @@ class OllamaAPIIntegrationTest { api.pullModel(VISION_MODEL); File imageFile = getImageFileFromClasspath("roses.jpg"); try { - OllamaResult result = api.generateWithImageFiles(VISION_MODEL, "What is in this image?", - List.of(imageFile), new OptionsBuilder().build()); + OllamaResult result = api.generateWithImages(VISION_MODEL, "What is in this image?", + List.of(imageFile), new OptionsBuilder().build(), null, null); assertNotNull(result); assertNotNull(result.getResponse()); assertFalse(result.getResponse().isEmpty()); @@ -637,11 +637,17 @@ class OllamaAPIIntegrationTest { StringBuffer sb = new StringBuffer(); - OllamaResult result = api.generateWithImageFiles(VISION_MODEL, "What is in this image?", - List.of(imageFile), new OptionsBuilder().build(), (s) -> { + OllamaResult result = api.generateWithImages( + VISION_MODEL, + "What is in this image?", + List.of(imageFile), + new OptionsBuilder().build(), + null, + (s) -> { LOG.info(s); sb.append(s); - }); + } + ); assertNotNull(result); assertNotNull(result.getResponse()); assertFalse(result.getResponse().isEmpty()); diff --git a/src/test/java/io/github/ollama4j/unittests/TestMockedAPIs.java b/src/test/java/io/github/ollama4j/unittests/TestMockedAPIs.java index f95a2dc..14471fa 100644 --- a/src/test/java/io/github/ollama4j/unittests/TestMockedAPIs.java +++ b/src/test/java/io/github/ollama4j/unittests/TestMockedAPIs.java @@ -154,15 +154,15 @@ class TestMockedAPIs { String model = OllamaModelType.LLAMA2; String prompt = "some prompt text"; try { - when(ollamaAPI.generateWithImageFiles( - model, prompt, Collections.emptyList(), new OptionsBuilder().build())) + when(ollamaAPI.generateWithImages( + model, prompt, Collections.emptyList(), new OptionsBuilder().build(), null, null)) .thenReturn(new OllamaResult("", "", 0, 200)); - ollamaAPI.generateWithImageFiles( - model, prompt, Collections.emptyList(), new OptionsBuilder().build()); + ollamaAPI.generateWithImages( + model, prompt, Collections.emptyList(), new OptionsBuilder().build(), null, null); verify(ollamaAPI, times(1)) - .generateWithImageFiles( - model, prompt, Collections.emptyList(), new OptionsBuilder().build()); - } catch (IOException | OllamaBaseException | InterruptedException e) { + .generateWithImages( + model, prompt, Collections.emptyList(), new OptionsBuilder().build(), null, null); + } catch (Exception e) { throw new RuntimeException(e); } } @@ -173,14 +173,14 @@ class TestMockedAPIs { String model = OllamaModelType.LLAMA2; String prompt = "some prompt text"; try { - when(ollamaAPI.generateWithImageURLs( - model, prompt, Collections.emptyList(), new OptionsBuilder().build())) + when(ollamaAPI.generateWithImages( + model, prompt, Collections.emptyList(), new OptionsBuilder().build(), null, null)) .thenReturn(new OllamaResult("", "", 0, 200)); - ollamaAPI.generateWithImageURLs( - model, prompt, Collections.emptyList(), new OptionsBuilder().build()); + ollamaAPI.generateWithImages( + model, prompt, Collections.emptyList(), new OptionsBuilder().build(), null, null); verify(ollamaAPI, times(1)) - .generateWithImageURLs( - model, prompt, Collections.emptyList(), new OptionsBuilder().build()); + .generateWithImages( + model, prompt, Collections.emptyList(), new OptionsBuilder().build(), null, null); } catch (IOException | OllamaBaseException | InterruptedException | URISyntaxException e) { throw new RuntimeException(e); } @@ -191,10 +191,10 @@ class TestMockedAPIs { OllamaAPI ollamaAPI = Mockito.mock(OllamaAPI.class); String model = OllamaModelType.LLAMA2; String prompt = "some prompt text"; - when(ollamaAPI.generateAsync(model, prompt, false, false)) + when(ollamaAPI.generate(model, prompt, false, false)) .thenReturn(new OllamaAsyncResultStreamer(null, null, 3)); - ollamaAPI.generateAsync(model, prompt, false, false); - verify(ollamaAPI, times(1)).generateAsync(model, prompt, false, false); + ollamaAPI.generate(model, prompt, false, false); + verify(ollamaAPI, times(1)).generate(model, prompt, false, false); } @Test diff --git a/src/test/java/io/github/ollama4j/unittests/jackson/TestGenerateRequestSerialization.java b/src/test/java/io/github/ollama4j/unittests/jackson/TestGenerateRequestSerialization.java index bf9b970..f6c1da9 100644 --- a/src/test/java/io/github/ollama4j/unittests/jackson/TestGenerateRequestSerialization.java +++ b/src/test/java/io/github/ollama4j/unittests/jackson/TestGenerateRequestSerialization.java @@ -44,11 +44,11 @@ public class TestGenerateRequestSerialization extends AbstractSerializationTest< builder.withPrompt("Some prompt").withGetJsonResponse().build(); String jsonRequest = serialize(req); + System.out.printf(jsonRequest); // no jackson deserialization as format property is not boolean ==> omit as deserialization // of request is never used in real code anyways JSONObject jsonObject = new JSONObject(jsonRequest); String requestFormatProperty = jsonObject.getString("format"); assertEquals("json", requestFormatProperty); } - }