From 376938653955ca5780f06f5446a67277986c9fb8 Mon Sep 17 00:00:00 2001 From: Markus Klenke Date: Mon, 12 Feb 2024 22:08:10 +0000 Subject: [PATCH] Changes images property of ChatMessage to type byte[] --- .../core/models/chat/OllamaChatMessage.java | 2 +- .../models/chat/OllamaChatRequestBuilder.java | 50 ++++++++++++++++++- .../core/utils/FileToBase64Serializer.java | 9 ++-- 3 files changed, 53 insertions(+), 8 deletions(-) diff --git a/src/main/java/io/github/amithkoujalgi/ollama4j/core/models/chat/OllamaChatMessage.java b/src/main/java/io/github/amithkoujalgi/ollama4j/core/models/chat/OllamaChatMessage.java index 85d2a17..3d0d932 100644 --- a/src/main/java/io/github/amithkoujalgi/ollama4j/core/models/chat/OllamaChatMessage.java +++ b/src/main/java/io/github/amithkoujalgi/ollama4j/core/models/chat/OllamaChatMessage.java @@ -33,7 +33,7 @@ public class OllamaChatMessage { private String content; @JsonSerialize(using = FileToBase64Serializer.class) - private List images; + private List images; @Override public String toString() { diff --git a/src/main/java/io/github/amithkoujalgi/ollama4j/core/models/chat/OllamaChatRequestBuilder.java b/src/main/java/io/github/amithkoujalgi/ollama4j/core/models/chat/OllamaChatRequestBuilder.java index af28b7b..9d15d19 100644 --- a/src/main/java/io/github/amithkoujalgi/ollama4j/core/models/chat/OllamaChatRequestBuilder.java +++ b/src/main/java/io/github/amithkoujalgi/ollama4j/core/models/chat/OllamaChatRequestBuilder.java @@ -1,16 +1,26 @@ package io.github.amithkoujalgi.ollama4j.core.models.chat; import java.io.File; +import java.io.IOException; +import java.net.URISyntaxException; +import java.nio.file.Files; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import io.github.amithkoujalgi.ollama4j.core.utils.Options; +import io.github.amithkoujalgi.ollama4j.core.utils.Utils; /** * Helper class for creating {@link OllamaChatRequestModel} objects using the builder-pattern. */ public class OllamaChatRequestBuilder { + private static final Logger LOG = LoggerFactory.getLogger(OllamaChatRequestBuilder.class); + private OllamaChatRequestBuilder(String model, List messages){ request = new OllamaChatRequestModel(model, messages); } @@ -29,9 +39,45 @@ public class OllamaChatRequestBuilder { request = new OllamaChatRequestModel(request.getModel(), new ArrayList<>()); } - public OllamaChatRequestBuilder withMessage(OllamaChatMessageRole role, String content, File... images){ + public OllamaChatRequestBuilder withMessage(OllamaChatMessageRole role, String content){ + return withMessage(role, content, (String)null); + } + + public OllamaChatRequestBuilder withMessage(OllamaChatMessageRole role, String content, List images){ List messages = this.request.getMessages(); - messages.add(new OllamaChatMessage(role,content,List.of(images))); + + List binaryImages = images.stream().map(file -> { + try { + return Files.readAllBytes(file.toPath()); + } catch (IOException e) { + LOG.warn(String.format("File '%s' could not be accessed, will not add to message!",file.toPath()), e); + return new byte[0]; + } + }).collect(Collectors.toList()); + + messages.add(new OllamaChatMessage(role,content,binaryImages)); + return this; + } + + public OllamaChatRequestBuilder withMessage(OllamaChatMessageRole role, String content, String... imageUrls){ + List messages = this.request.getMessages(); + List binaryImages = null; + if(imageUrls.length>0){ + binaryImages = new ArrayList<>(); + for (String imageUrl : imageUrls) { + try{ + binaryImages.add(Utils.loadImageBytesFromUrl(imageUrl)); + } + catch (URISyntaxException e){ + LOG.warn(String.format("URL '%s' could not be accessed, will not add to message!",imageUrl), e); + } + catch (IOException e){ + LOG.warn(String.format("Content of URL '%s' could not be read, will not add to message!",imageUrl), e); + } + } + } + + messages.add(new OllamaChatMessage(role,content,binaryImages)); return this; } diff --git a/src/main/java/io/github/amithkoujalgi/ollama4j/core/utils/FileToBase64Serializer.java b/src/main/java/io/github/amithkoujalgi/ollama4j/core/utils/FileToBase64Serializer.java index 8151963..680635b 100644 --- a/src/main/java/io/github/amithkoujalgi/ollama4j/core/utils/FileToBase64Serializer.java +++ b/src/main/java/io/github/amithkoujalgi/ollama4j/core/utils/FileToBase64Serializer.java @@ -1,7 +1,6 @@ package io.github.amithkoujalgi.ollama4j.core.utils; import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.IOException; import java.io.ObjectOutputStream; import java.util.Base64; @@ -11,13 +10,13 @@ import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.databind.JsonSerializer; import com.fasterxml.jackson.databind.SerializerProvider; -public class FileToBase64Serializer extends JsonSerializer> { +public class FileToBase64Serializer extends JsonSerializer> { @Override - public void serialize(Collection value, JsonGenerator jsonGenerator, SerializerProvider serializers) throws IOException { + public void serialize(Collection value, JsonGenerator jsonGenerator, SerializerProvider serializers) throws IOException { jsonGenerator.writeStartArray(); - for (File file : value) { - jsonGenerator.writeString(Base64.getEncoder().encodeToString(serialize(file))); + for (byte[] file : value) { + jsonGenerator.writeString(Base64.getEncoder().encodeToString(file)); } jsonGenerator.writeEndArray(); }