diff --git a/src/main/java/app/mealsmadeeasy/api/inference/InferenceController.java b/src/main/java/app/mealsmadeeasy/api/inference/InferenceController.java new file mode 100644 index 0000000..471aa68 --- /dev/null +++ b/src/main/java/app/mealsmadeeasy/api/inference/InferenceController.java @@ -0,0 +1,37 @@ +package app.mealsmadeeasy.api.inference; + +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.http.codec.ServerSentEvent; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.multipart.MultipartFile; +import reactor.core.publisher.Flux; + +import java.io.IOException; +import java.util.Map; + +@RestController +@RequestMapping("/inferences") +public class InferenceController { + + private final InferenceService inferenceService; + + public InferenceController(InferenceService inferenceService) { + this.inferenceService = inferenceService; + } + + @PutMapping("/recipe-extract") + public ResponseEntity recipeExtract(@RequestParam MultipartFile recipeImageFile) throws IOException { + return ResponseEntity.ok(this.inferenceService.extractRecipe(recipeImageFile.getInputStream())); + } + + @PutMapping(value = "/recipe-extract-stream", produces = MediaType.TEXT_EVENT_STREAM_VALUE) + public Flux>> recipeExtractStream(@RequestParam MultipartFile recipeImageFile) throws IOException { + return this.inferenceService.extractRecipeStream(recipeImageFile.getInputStream()) + .map(data -> ServerSentEvent.builder(Map.of("delta", data)).build()); + } + +} diff --git a/src/main/java/app/mealsmadeeasy/api/inference/InferenceService.java b/src/main/java/app/mealsmadeeasy/api/inference/InferenceService.java new file mode 100644 index 0000000..8b9973e --- /dev/null +++ b/src/main/java/app/mealsmadeeasy/api/inference/InferenceService.java @@ -0,0 +1,52 @@ +package app.mealsmadeeasy.api.inference; + +import org.springframework.ai.chat.client.ChatClient; +import org.springframework.ai.content.Media; +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.InputStreamResource; +import org.springframework.stereotype.Service; +import org.springframework.util.MimeType; +import reactor.core.publisher.Flux; + +import java.io.InputStream; + +@Service +public class InferenceService { + + private final ChatClient chatClient; + + public InferenceService(ChatClient.Builder chatClientBuilder) { + this.chatClient = chatClientBuilder.build(); + } + + public String extractRecipe(InputStream recipeImageInputStream) { + final Media media = Media.builder() + .data(new InputStreamResource(recipeImageInputStream)) + .mimeType(MimeType.valueOf("image/jpeg")) + .build(); + + final String markdownResponse = this.chatClient.prompt() + .user(u -> + u.text(new ClassPathResource("app/mealsmadeeasy/api/inference/recipe-extract-user-prompt.md")) + .media(media) + ) + .call() + .content(); + return markdownResponse; + } + + public Flux extractRecipeStream(InputStream recipeImageInputStream) { + final Media media = Media.builder() + .data(new InputStreamResource(recipeImageInputStream)) + .mimeType(MimeType.valueOf("image/jpeg")) + .build(); + + return this.chatClient.prompt() + .user(u -> + u.text(new ClassPathResource("app/mealsmadeeasy/api/inference/recipe-extract-user-prompt.md")) + .media(media) + ) + .stream().content(); + } + +} diff --git a/src/main/java/app/mealsmadeeasy/api/inference/RecipeExtractionEntity.java b/src/main/java/app/mealsmadeeasy/api/inference/RecipeExtractionEntity.java new file mode 100644 index 0000000..5eb4f46 --- /dev/null +++ b/src/main/java/app/mealsmadeeasy/api/inference/RecipeExtractionEntity.java @@ -0,0 +1,36 @@ +package app.mealsmadeeasy.api.inference; + +import jakarta.persistence.*; + +import java.util.UUID; + +@Entity +@Table(name = "recipe_extraction") +public class RecipeExtractionEntity { + + @Id + @GeneratedValue(strategy = GenerationType.UUID) + private UUID id; + + @Lob + @Column(columnDefinition = "TEXT") + @Basic(fetch = FetchType.LAZY) + private String markdown; + + public UUID getId() { + return this.id; + } + + public void setId(UUID id) { + this.id = id; + } + + public String getMarkdown() { + return this.markdown; + } + + public void setMarkdown(String markdown) { + this.markdown = markdown; + } + +} diff --git a/src/main/java/app/mealsmadeeasy/api/inference/RecipeExtractionView.java b/src/main/java/app/mealsmadeeasy/api/inference/RecipeExtractionView.java new file mode 100644 index 0000000..3f47bbc --- /dev/null +++ b/src/main/java/app/mealsmadeeasy/api/inference/RecipeExtractionView.java @@ -0,0 +1,33 @@ +package app.mealsmadeeasy.api.inference; + +import java.util.UUID; + +public class RecipeExtractionView { + + public static RecipeExtractionView from(RecipeExtractionEntity entity) { + final var view = new RecipeExtractionView(); + view.setId(entity.getId()); + view.setMarkdown(entity.getMarkdown()); + return view; + } + + private UUID id; + private String markdown; + + public UUID getId() { + return this.id; + } + + public void setId(UUID id) { + this.id = id; + } + + public String getMarkdown() { + return this.markdown; + } + + public void setMarkdown(String markdown) { + this.markdown = markdown; + } + +} diff --git a/src/main/resources/app/mealsmadeeasy/api/inference/recipe-extract-user-prompt.md b/src/main/resources/app/mealsmadeeasy/api/inference/recipe-extract-user-prompt.md new file mode 100644 index 0000000..f08842d --- /dev/null +++ b/src/main/resources/app/mealsmadeeasy/api/inference/recipe-extract-user-prompt.md @@ -0,0 +1 @@ +Convert the recipe in the image to Markdown. \ No newline at end of file diff --git a/src/main/resources/application.properties b/src/main/resources/application.properties index 1e6e764..0a5d6ba 100644 --- a/src/main/resources/application.properties +++ b/src/main/resources/application.properties @@ -19,3 +19,5 @@ app.mealsmadeeasy.api.images.bucketName=images # AI spring.ai.vectorstore.pgvector.dimensions=1024 +spring.ai.ollama.chat.options.model=deepseek-ocr:latest +spring.ai.ollama.init.pull-model-strategy=never \ No newline at end of file