Add second AI call to recipe infer job.

This commit is contained in:
Jesse Brault 2026-01-16 23:01:34 -06:00
parent b19dc42094
commit 547c04fbab
5 changed files with 122 additions and 4 deletions

View File

@ -75,6 +75,9 @@ public class RecipeInferJobIntegrationTests {
final RecipeDraft draftWithInference = this.recipeService.getDraftById(draft.getId());
assertThat(draftWithInference.getTitle(), is(notNullValue()));
assertThat(draftWithInference.getIngredients(), is(notNullValue()));
assertThat(draftWithInference.getInferences(), is(notNullValue()));
final List<RecipeDraft.RecipeDraftInference> inferences = draftWithInference.getInferences();

View File

@ -11,3 +11,5 @@ app.mealsmadeeasy.api.files.bucketName=files
# Posted by Iogui, modified by community. See post 'Timeline' for change history
# Retrieved 2025-12-25, License - CC BY-SA 4.0
spring.datasource.hikari.auto-commit=false
logging.level.app.mealsmadeeasy.api=debug

View File

@ -29,6 +29,13 @@ public class RecipeDraft {
private String rawText;
}
@Data
public static class IngredientDraft {
private @Nullable String amount;
private String name;
private @Nullable String notes;
}
@Id
@GeneratedValue(strategy = GenerationType.UUID)
@Column(nullable = false, unique = true, updatable = false)
@ -48,6 +55,10 @@ public class RecipeDraft {
private @Nullable Integer totalTime;
private @Nullable String rawText;
@Type(JsonBinaryType.class)
@Column(columnDefinition = "jsonb")
private @Nullable List<IngredientDraft> ingredients;
@ManyToOne(optional = false)
@JoinColumn(name = "owner_id", nullable = false)
private User owner;

View File

@ -6,14 +6,20 @@ import app.mealsmadeeasy.api.job.Job;
import app.mealsmadeeasy.api.job.JobHandler;
import app.mealsmadeeasy.api.recipe.RecipeDraft;
import app.mealsmadeeasy.api.recipe.RecipeService;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.RequiredArgsConstructor;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.ai.chat.messages.Message;
import org.springframework.ai.chat.messages.SystemMessage;
import org.springframework.ai.chat.messages.UserMessage;
import org.springframework.ai.chat.model.ChatModel;
import org.springframework.ai.chat.model.ChatResponse;
import org.springframework.ai.chat.prompt.ChatOptions;
import org.springframework.ai.chat.prompt.Prompt;
import org.springframework.ai.content.Media;
import org.springframework.ai.converter.BeanOutputConverter;
import org.springframework.ai.ollama.api.OllamaChatOptions;
import org.springframework.core.io.InputStreamResource;
import org.springframework.stereotype.Component;
@ -23,16 +29,32 @@ import java.io.IOException;
import java.io.InputStream;
import java.time.OffsetDateTime;
import java.util.ArrayList;
import java.util.List;
@Component
@RequiredArgsConstructor
public class RecipeInferJobHandler implements JobHandler<RecipeInferJobPayload> {
private record RecipeExtraction(
@JsonProperty(required = true) String title,
@JsonProperty(required = true) List<IngredientExtraction> ingredients
) {}
private record IngredientExtraction(
String amount,
@JsonProperty(required = true) String name,
String notes
) {}
public static final String JOB_KEY = "RECIPE_INFER_JOB";
private static final Logger logger = LoggerFactory.getLogger(RecipeInferJobHandler.class);
private final FileService fileService;
private final RecipeService recipeService;
private final ChatModel chatModel;
private final BeanOutputConverter<RecipeExtraction> extractionConverter =
new BeanOutputConverter<>(RecipeExtraction.class);
@Override
public Class<RecipeInferJobPayload> getPayloadType() {
@ -46,6 +68,7 @@ public class RecipeInferJobHandler implements JobHandler<RecipeInferJobPayload>
@Override
public void handle(Job job, RecipeInferJobPayload payload) {
logger.debug("Starting recipe inference job {}", job.getId());
final File sourceFile = this.fileService.getById(payload.fileId());
final InputStream sourceFileContent;
try {
@ -54,13 +77,14 @@ public class RecipeInferJobHandler implements JobHandler<RecipeInferJobPayload>
throw new RuntimeException(e);
}
// 1. OCR of recipe image
final Media sourceFileMedia = Media.builder()
.data(new InputStreamResource(sourceFileContent))
.mimeType(MimeType.valueOf(sourceFile.getMimeType()))
.build();
final Message ocrMessage = UserMessage.builder()
.text("Convert the recipe in the image to Markdown.")
.text("Convert the recipe in the given file to Markdown.")
.media(sourceFileMedia)
.build();
final Prompt ocrPrompt = Prompt.builder()
@ -69,26 +93,103 @@ public class RecipeInferJobHandler implements JobHandler<RecipeInferJobPayload>
.build();
final ChatResponse ocrResponse = this.chatModel.call(ocrPrompt);
final String fullMarkdownText = ocrResponse.getResult().getOutput().getText();
if (fullMarkdownText == null) {
throw new RuntimeException("fullMarkdownText from ocr came back null");
}
// get recipe draft
logger.debug("ocr returned:\n{}", fullMarkdownText);
// 2. Extract title and ingredients
final Message extractSystemMessage = SystemMessage.builder()
.text("""
Extract from the given Markdown food recipe the title and ingredients.
For each ingredient, you must provide the name; amount and notes are optional.
Here is an example of a full extraction:
```json
{
"title": "Delicious Recipe",
"ingredients": [
{
"amount": "1 tablespoon",
"name": "garlic powder"
},
{
"amount": "1 lb",
"name": "tofu",
"notes": "pressed and dried"
},
{
"name": "salt and pepper"
}
]
}
```
""".stripIndent().trim()
)
.build();
final Message extractUserMessage = UserMessage.builder()
.text(fullMarkdownText)
.build();
final ChatOptions extractChatOptions = OllamaChatOptions.builder()
.model("gemma3:latest")
.format(extractionConverter.getJsonSchemaMap())
.build();
final Prompt extractPrompt = Prompt.builder()
.messages(extractSystemMessage, extractUserMessage)
.chatOptions(extractChatOptions)
.build();
final ChatResponse extractResponse = this.chatModel.call(extractPrompt);
final String extractContent = extractResponse.getResult().getOutput().getText();
if (extractContent == null) {
throw new RuntimeException("extractContent from extract came back null");
}
logger.debug("extract returned:\n{}", extractContent);
final RecipeExtraction extraction = this.extractionConverter.convert(extractContent);
// 3. Get draft and set props from extraction
final @Nullable RecipeDraft recipeDraft = this.recipeService.getDraftById(payload.recipeDraftId());
if (recipeDraft == null) {
throw new RuntimeException("Recipe draft not found");
}
// set props on draft from ai output, etc.
recipeDraft.setTitle(extraction.title());
// todo: calculate slug
// ingredients
if (recipeDraft.getIngredients() == null) {
recipeDraft.setIngredients(new ArrayList<>());
}
recipeDraft.getIngredients().addAll(extraction.ingredients().stream()
.map(extractedIngredient -> {
final RecipeDraft.IngredientDraft ingredientDraft = new RecipeDraft.IngredientDraft();
ingredientDraft.setAmount(extractedIngredient.amount());
ingredientDraft.setName(extractedIngredient.name());
ingredientDraft.setNotes(extractedIngredient.notes());
return ingredientDraft;
})
.toList()
);
// other props
recipeDraft.setRawText(fullMarkdownText);
recipeDraft.setState(RecipeDraft.State.ENTER_DATA);
// save inference for later
if (recipeDraft.getInferences() == null) {
recipeDraft.setInferences(new ArrayList<>());
}
final RecipeDraft.RecipeDraftInference inference = new RecipeDraft.RecipeDraftInference();
inference.setTitle("TODO: inferred title");
inference.setTitle(extraction.title());
inference.setRawText(fullMarkdownText);
inference.setInferredAt(OffsetDateTime.now());
recipeDraft.getInferences().add(inference);
logger.debug("Recipe infer job completed, saving to db...");
this.recipeService.saveDraft(recipeDraft);
}

View File

@ -0,0 +1 @@
ALTER TABLE recipe_draft ADD COLUMN ingredients JSONB;