diff --git a/src/integrationTest/java/app/mealsmadeeasy/api/recipe/job/RecipeInferJobIntegrationTests.java b/src/integrationTest/java/app/mealsmadeeasy/api/recipe/job/RecipeInferJobIntegrationTests.java index d7da271..71d580b 100644 --- a/src/integrationTest/java/app/mealsmadeeasy/api/recipe/job/RecipeInferJobIntegrationTests.java +++ b/src/integrationTest/java/app/mealsmadeeasy/api/recipe/job/RecipeInferJobIntegrationTests.java @@ -75,6 +75,9 @@ public class RecipeInferJobIntegrationTests { final RecipeDraft draftWithInference = this.recipeService.getDraftById(draft.getId()); + assertThat(draftWithInference.getTitle(), is(notNullValue())); + assertThat(draftWithInference.getIngredients(), is(notNullValue())); + assertThat(draftWithInference.getInferences(), is(notNullValue())); final List inferences = draftWithInference.getInferences(); diff --git a/src/integrationTest/resources/application.properties b/src/integrationTest/resources/application.properties index 283333e..1341b82 100644 --- a/src/integrationTest/resources/application.properties +++ b/src/integrationTest/resources/application.properties @@ -11,3 +11,5 @@ app.mealsmadeeasy.api.files.bucketName=files # Posted by Iogui, modified by community. See post 'Timeline' for change history # Retrieved 2025-12-25, License - CC BY-SA 4.0 spring.datasource.hikari.auto-commit=false + +logging.level.app.mealsmadeeasy.api=debug \ No newline at end of file diff --git a/src/main/java/app/mealsmadeeasy/api/recipe/RecipeDraft.java b/src/main/java/app/mealsmadeeasy/api/recipe/RecipeDraft.java index 6b8dcdc..17e697d 100644 --- a/src/main/java/app/mealsmadeeasy/api/recipe/RecipeDraft.java +++ b/src/main/java/app/mealsmadeeasy/api/recipe/RecipeDraft.java @@ -29,6 +29,13 @@ public class RecipeDraft { private String rawText; } + @Data + public static class IngredientDraft { + private @Nullable String amount; + private String name; + private @Nullable String notes; + } + @Id @GeneratedValue(strategy = GenerationType.UUID) @Column(nullable = false, unique = true, updatable = false) @@ -48,6 +55,10 @@ public class RecipeDraft { private @Nullable Integer totalTime; private @Nullable String rawText; + @Type(JsonBinaryType.class) + @Column(columnDefinition = "jsonb") + private @Nullable List ingredients; + @ManyToOne(optional = false) @JoinColumn(name = "owner_id", nullable = false) private User owner; diff --git a/src/main/java/app/mealsmadeeasy/api/recipe/job/RecipeInferJobHandler.java b/src/main/java/app/mealsmadeeasy/api/recipe/job/RecipeInferJobHandler.java index 8a8f1bc..861a076 100644 --- a/src/main/java/app/mealsmadeeasy/api/recipe/job/RecipeInferJobHandler.java +++ b/src/main/java/app/mealsmadeeasy/api/recipe/job/RecipeInferJobHandler.java @@ -6,14 +6,20 @@ import app.mealsmadeeasy.api.job.Job; import app.mealsmadeeasy.api.job.JobHandler; import app.mealsmadeeasy.api.recipe.RecipeDraft; import app.mealsmadeeasy.api.recipe.RecipeService; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.RequiredArgsConstructor; import org.jetbrains.annotations.Nullable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.ai.chat.messages.Message; +import org.springframework.ai.chat.messages.SystemMessage; import org.springframework.ai.chat.messages.UserMessage; import org.springframework.ai.chat.model.ChatModel; import org.springframework.ai.chat.model.ChatResponse; +import org.springframework.ai.chat.prompt.ChatOptions; import org.springframework.ai.chat.prompt.Prompt; import org.springframework.ai.content.Media; +import org.springframework.ai.converter.BeanOutputConverter; import org.springframework.ai.ollama.api.OllamaChatOptions; import org.springframework.core.io.InputStreamResource; import org.springframework.stereotype.Component; @@ -23,16 +29,32 @@ import java.io.IOException; import java.io.InputStream; import java.time.OffsetDateTime; import java.util.ArrayList; +import java.util.List; @Component @RequiredArgsConstructor public class RecipeInferJobHandler implements JobHandler { + private record RecipeExtraction( + @JsonProperty(required = true) String title, + @JsonProperty(required = true) List ingredients + ) {} + + private record IngredientExtraction( + String amount, + @JsonProperty(required = true) String name, + String notes + ) {} + public static final String JOB_KEY = "RECIPE_INFER_JOB"; + private static final Logger logger = LoggerFactory.getLogger(RecipeInferJobHandler.class); + private final FileService fileService; private final RecipeService recipeService; private final ChatModel chatModel; + private final BeanOutputConverter extractionConverter = + new BeanOutputConverter<>(RecipeExtraction.class); @Override public Class getPayloadType() { @@ -46,6 +68,7 @@ public class RecipeInferJobHandler implements JobHandler @Override public void handle(Job job, RecipeInferJobPayload payload) { + logger.debug("Starting recipe inference job {}", job.getId()); final File sourceFile = this.fileService.getById(payload.fileId()); final InputStream sourceFileContent; try { @@ -54,13 +77,14 @@ public class RecipeInferJobHandler implements JobHandler throw new RuntimeException(e); } + // 1. OCR of recipe image final Media sourceFileMedia = Media.builder() .data(new InputStreamResource(sourceFileContent)) .mimeType(MimeType.valueOf(sourceFile.getMimeType())) .build(); final Message ocrMessage = UserMessage.builder() - .text("Convert the recipe in the image to Markdown.") + .text("Convert the recipe in the given file to Markdown.") .media(sourceFileMedia) .build(); final Prompt ocrPrompt = Prompt.builder() @@ -69,26 +93,103 @@ public class RecipeInferJobHandler implements JobHandler .build(); final ChatResponse ocrResponse = this.chatModel.call(ocrPrompt); final String fullMarkdownText = ocrResponse.getResult().getOutput().getText(); + if (fullMarkdownText == null) { + throw new RuntimeException("fullMarkdownText from ocr came back null"); + } - // get recipe draft + logger.debug("ocr returned:\n{}", fullMarkdownText); + + // 2. Extract title and ingredients + final Message extractSystemMessage = SystemMessage.builder() + .text(""" + Extract from the given Markdown food recipe the title and ingredients. + For each ingredient, you must provide the name; amount and notes are optional. + + Here is an example of a full extraction: + + ```json + { + "title": "Delicious Recipe", + "ingredients": [ + { + "amount": "1 tablespoon", + "name": "garlic powder" + }, + { + "amount": "1 lb", + "name": "tofu", + "notes": "pressed and dried" + }, + { + "name": "salt and pepper" + } + ] + } + ``` + """.stripIndent().trim() + ) + .build(); + final Message extractUserMessage = UserMessage.builder() + .text(fullMarkdownText) + .build(); + final ChatOptions extractChatOptions = OllamaChatOptions.builder() + .model("gemma3:latest") + .format(extractionConverter.getJsonSchemaMap()) + .build(); + final Prompt extractPrompt = Prompt.builder() + .messages(extractSystemMessage, extractUserMessage) + .chatOptions(extractChatOptions) + .build(); + final ChatResponse extractResponse = this.chatModel.call(extractPrompt); + final String extractContent = extractResponse.getResult().getOutput().getText(); + if (extractContent == null) { + throw new RuntimeException("extractContent from extract came back null"); + } + logger.debug("extract returned:\n{}", extractContent); + + final RecipeExtraction extraction = this.extractionConverter.convert(extractContent); + + // 3. Get draft and set props from extraction final @Nullable RecipeDraft recipeDraft = this.recipeService.getDraftById(payload.recipeDraftId()); if (recipeDraft == null) { throw new RuntimeException("Recipe draft not found"); } - // set props on draft from ai output, etc. + recipeDraft.setTitle(extraction.title()); + + // todo: calculate slug + + // ingredients + if (recipeDraft.getIngredients() == null) { + recipeDraft.setIngredients(new ArrayList<>()); + } + recipeDraft.getIngredients().addAll(extraction.ingredients().stream() + .map(extractedIngredient -> { + final RecipeDraft.IngredientDraft ingredientDraft = new RecipeDraft.IngredientDraft(); + ingredientDraft.setAmount(extractedIngredient.amount()); + ingredientDraft.setName(extractedIngredient.name()); + ingredientDraft.setNotes(extractedIngredient.notes()); + return ingredientDraft; + }) + .toList() + ); + + // other props recipeDraft.setRawText(fullMarkdownText); recipeDraft.setState(RecipeDraft.State.ENTER_DATA); + // save inference for later if (recipeDraft.getInferences() == null) { recipeDraft.setInferences(new ArrayList<>()); } final RecipeDraft.RecipeDraftInference inference = new RecipeDraft.RecipeDraftInference(); - inference.setTitle("TODO: inferred title"); + inference.setTitle(extraction.title()); inference.setRawText(fullMarkdownText); inference.setInferredAt(OffsetDateTime.now()); recipeDraft.getInferences().add(inference); + logger.debug("Recipe infer job completed, saving to db..."); + this.recipeService.saveDraft(recipeDraft); } diff --git a/src/main/resources/db/migration/V8__add_ingredients_to_recipe_draft.sql b/src/main/resources/db/migration/V8__add_ingredients_to_recipe_draft.sql new file mode 100644 index 0000000..4b46422 --- /dev/null +++ b/src/main/resources/db/migration/V8__add_ingredients_to_recipe_draft.sql @@ -0,0 +1 @@ +ALTER TABLE recipe_draft ADD COLUMN ingredients JSONB; \ No newline at end of file