Add second AI call to recipe infer job.

This commit is contained in:
Jesse Brault 2026-01-16 23:01:34 -06:00
parent b19dc42094
commit 547c04fbab
5 changed files with 122 additions and 4 deletions

View File

@ -75,6 +75,9 @@ public class RecipeInferJobIntegrationTests {
final RecipeDraft draftWithInference = this.recipeService.getDraftById(draft.getId()); final RecipeDraft draftWithInference = this.recipeService.getDraftById(draft.getId());
assertThat(draftWithInference.getTitle(), is(notNullValue()));
assertThat(draftWithInference.getIngredients(), is(notNullValue()));
assertThat(draftWithInference.getInferences(), is(notNullValue())); assertThat(draftWithInference.getInferences(), is(notNullValue()));
final List<RecipeDraft.RecipeDraftInference> inferences = draftWithInference.getInferences(); final List<RecipeDraft.RecipeDraftInference> inferences = draftWithInference.getInferences();

View File

@ -11,3 +11,5 @@ app.mealsmadeeasy.api.files.bucketName=files
# Posted by Iogui, modified by community. See post 'Timeline' for change history # Posted by Iogui, modified by community. See post 'Timeline' for change history
# Retrieved 2025-12-25, License - CC BY-SA 4.0 # Retrieved 2025-12-25, License - CC BY-SA 4.0
spring.datasource.hikari.auto-commit=false spring.datasource.hikari.auto-commit=false
logging.level.app.mealsmadeeasy.api=debug

View File

@ -29,6 +29,13 @@ public class RecipeDraft {
private String rawText; private String rawText;
} }
@Data
public static class IngredientDraft {
private @Nullable String amount;
private String name;
private @Nullable String notes;
}
@Id @Id
@GeneratedValue(strategy = GenerationType.UUID) @GeneratedValue(strategy = GenerationType.UUID)
@Column(nullable = false, unique = true, updatable = false) @Column(nullable = false, unique = true, updatable = false)
@ -48,6 +55,10 @@ public class RecipeDraft {
private @Nullable Integer totalTime; private @Nullable Integer totalTime;
private @Nullable String rawText; private @Nullable String rawText;
@Type(JsonBinaryType.class)
@Column(columnDefinition = "jsonb")
private @Nullable List<IngredientDraft> ingredients;
@ManyToOne(optional = false) @ManyToOne(optional = false)
@JoinColumn(name = "owner_id", nullable = false) @JoinColumn(name = "owner_id", nullable = false)
private User owner; private User owner;

View File

@ -6,14 +6,20 @@ import app.mealsmadeeasy.api.job.Job;
import app.mealsmadeeasy.api.job.JobHandler; import app.mealsmadeeasy.api.job.JobHandler;
import app.mealsmadeeasy.api.recipe.RecipeDraft; import app.mealsmadeeasy.api.recipe.RecipeDraft;
import app.mealsmadeeasy.api.recipe.RecipeService; import app.mealsmadeeasy.api.recipe.RecipeService;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.RequiredArgsConstructor; import lombok.RequiredArgsConstructor;
import org.jetbrains.annotations.Nullable; import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.chat.messages.Message; import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.SystemMessage;
import org.springframework.ai.chat.messages.UserMessage; import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.chat.model.ChatModel; import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.model.ChatResponse; import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.prompt.ChatOptions;
import org.springframework.ai.chat.prompt.Prompt; import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.content.Media; import org.springframework.ai.content.Media;
import org.springframework.ai.converter.BeanOutputConverter;
import org.springframework.ai.ollama.api.OllamaChatOptions; import org.springframework.ai.ollama.api.OllamaChatOptions;
import org.springframework.core.io.InputStreamResource; import org.springframework.core.io.InputStreamResource;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
@ -23,16 +29,32 @@ import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.time.OffsetDateTime; import java.time.OffsetDateTime;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List;
@Component @Component
@RequiredArgsConstructor @RequiredArgsConstructor
public class RecipeInferJobHandler implements JobHandler<RecipeInferJobPayload> { public class RecipeInferJobHandler implements JobHandler<RecipeInferJobPayload> {
private record RecipeExtraction(
@JsonProperty(required = true) String title,
@JsonProperty(required = true) List<IngredientExtraction> ingredients
) {}
private record IngredientExtraction(
String amount,
@JsonProperty(required = true) String name,
String notes
) {}
public static final String JOB_KEY = "RECIPE_INFER_JOB"; public static final String JOB_KEY = "RECIPE_INFER_JOB";
private static final Logger logger = LoggerFactory.getLogger(RecipeInferJobHandler.class);
private final FileService fileService; private final FileService fileService;
private final RecipeService recipeService; private final RecipeService recipeService;
private final ChatModel chatModel; private final ChatModel chatModel;
private final BeanOutputConverter<RecipeExtraction> extractionConverter =
new BeanOutputConverter<>(RecipeExtraction.class);
@Override @Override
public Class<RecipeInferJobPayload> getPayloadType() { public Class<RecipeInferJobPayload> getPayloadType() {
@ -46,6 +68,7 @@ public class RecipeInferJobHandler implements JobHandler<RecipeInferJobPayload>
@Override @Override
public void handle(Job job, RecipeInferJobPayload payload) { public void handle(Job job, RecipeInferJobPayload payload) {
logger.debug("Starting recipe inference job {}", job.getId());
final File sourceFile = this.fileService.getById(payload.fileId()); final File sourceFile = this.fileService.getById(payload.fileId());
final InputStream sourceFileContent; final InputStream sourceFileContent;
try { try {
@ -54,13 +77,14 @@ public class RecipeInferJobHandler implements JobHandler<RecipeInferJobPayload>
throw new RuntimeException(e); throw new RuntimeException(e);
} }
// 1. OCR of recipe image
final Media sourceFileMedia = Media.builder() final Media sourceFileMedia = Media.builder()
.data(new InputStreamResource(sourceFileContent)) .data(new InputStreamResource(sourceFileContent))
.mimeType(MimeType.valueOf(sourceFile.getMimeType())) .mimeType(MimeType.valueOf(sourceFile.getMimeType()))
.build(); .build();
final Message ocrMessage = UserMessage.builder() final Message ocrMessage = UserMessage.builder()
.text("Convert the recipe in the image to Markdown.") .text("Convert the recipe in the given file to Markdown.")
.media(sourceFileMedia) .media(sourceFileMedia)
.build(); .build();
final Prompt ocrPrompt = Prompt.builder() final Prompt ocrPrompt = Prompt.builder()
@ -69,26 +93,103 @@ public class RecipeInferJobHandler implements JobHandler<RecipeInferJobPayload>
.build(); .build();
final ChatResponse ocrResponse = this.chatModel.call(ocrPrompt); final ChatResponse ocrResponse = this.chatModel.call(ocrPrompt);
final String fullMarkdownText = ocrResponse.getResult().getOutput().getText(); final String fullMarkdownText = ocrResponse.getResult().getOutput().getText();
if (fullMarkdownText == null) {
throw new RuntimeException("fullMarkdownText from ocr came back null");
}
// get recipe draft logger.debug("ocr returned:\n{}", fullMarkdownText);
// 2. Extract title and ingredients
final Message extractSystemMessage = SystemMessage.builder()
.text("""
Extract from the given Markdown food recipe the title and ingredients.
For each ingredient, you must provide the name; amount and notes are optional.
Here is an example of a full extraction:
```json
{
"title": "Delicious Recipe",
"ingredients": [
{
"amount": "1 tablespoon",
"name": "garlic powder"
},
{
"amount": "1 lb",
"name": "tofu",
"notes": "pressed and dried"
},
{
"name": "salt and pepper"
}
]
}
```
""".stripIndent().trim()
)
.build();
final Message extractUserMessage = UserMessage.builder()
.text(fullMarkdownText)
.build();
final ChatOptions extractChatOptions = OllamaChatOptions.builder()
.model("gemma3:latest")
.format(extractionConverter.getJsonSchemaMap())
.build();
final Prompt extractPrompt = Prompt.builder()
.messages(extractSystemMessage, extractUserMessage)
.chatOptions(extractChatOptions)
.build();
final ChatResponse extractResponse = this.chatModel.call(extractPrompt);
final String extractContent = extractResponse.getResult().getOutput().getText();
if (extractContent == null) {
throw new RuntimeException("extractContent from extract came back null");
}
logger.debug("extract returned:\n{}", extractContent);
final RecipeExtraction extraction = this.extractionConverter.convert(extractContent);
// 3. Get draft and set props from extraction
final @Nullable RecipeDraft recipeDraft = this.recipeService.getDraftById(payload.recipeDraftId()); final @Nullable RecipeDraft recipeDraft = this.recipeService.getDraftById(payload.recipeDraftId());
if (recipeDraft == null) { if (recipeDraft == null) {
throw new RuntimeException("Recipe draft not found"); throw new RuntimeException("Recipe draft not found");
} }
// set props on draft from ai output, etc. recipeDraft.setTitle(extraction.title());
// todo: calculate slug
// ingredients
if (recipeDraft.getIngredients() == null) {
recipeDraft.setIngredients(new ArrayList<>());
}
recipeDraft.getIngredients().addAll(extraction.ingredients().stream()
.map(extractedIngredient -> {
final RecipeDraft.IngredientDraft ingredientDraft = new RecipeDraft.IngredientDraft();
ingredientDraft.setAmount(extractedIngredient.amount());
ingredientDraft.setName(extractedIngredient.name());
ingredientDraft.setNotes(extractedIngredient.notes());
return ingredientDraft;
})
.toList()
);
// other props
recipeDraft.setRawText(fullMarkdownText); recipeDraft.setRawText(fullMarkdownText);
recipeDraft.setState(RecipeDraft.State.ENTER_DATA); recipeDraft.setState(RecipeDraft.State.ENTER_DATA);
// save inference for later
if (recipeDraft.getInferences() == null) { if (recipeDraft.getInferences() == null) {
recipeDraft.setInferences(new ArrayList<>()); recipeDraft.setInferences(new ArrayList<>());
} }
final RecipeDraft.RecipeDraftInference inference = new RecipeDraft.RecipeDraftInference(); final RecipeDraft.RecipeDraftInference inference = new RecipeDraft.RecipeDraftInference();
inference.setTitle("TODO: inferred title"); inference.setTitle(extraction.title());
inference.setRawText(fullMarkdownText); inference.setRawText(fullMarkdownText);
inference.setInferredAt(OffsetDateTime.now()); inference.setInferredAt(OffsetDateTime.now());
recipeDraft.getInferences().add(inference); recipeDraft.getInferences().add(inference);
logger.debug("Recipe infer job completed, saving to db...");
this.recipeService.saveDraft(recipeDraft); this.recipeService.saveDraft(recipeDraft);
} }

View File

@ -0,0 +1 @@
ALTER TABLE recipe_draft ADD COLUMN ingredients JSONB;