Add second AI call to recipe infer job.
This commit is contained in:
parent
b19dc42094
commit
547c04fbab
@ -75,6 +75,9 @@ public class RecipeInferJobIntegrationTests {
|
||||
|
||||
final RecipeDraft draftWithInference = this.recipeService.getDraftById(draft.getId());
|
||||
|
||||
assertThat(draftWithInference.getTitle(), is(notNullValue()));
|
||||
assertThat(draftWithInference.getIngredients(), is(notNullValue()));
|
||||
|
||||
assertThat(draftWithInference.getInferences(), is(notNullValue()));
|
||||
final List<RecipeDraft.RecipeDraftInference> inferences = draftWithInference.getInferences();
|
||||
|
||||
|
||||
@ -11,3 +11,5 @@ app.mealsmadeeasy.api.files.bucketName=files
|
||||
# Posted by Iogui, modified by community. See post 'Timeline' for change history
|
||||
# Retrieved 2025-12-25, License - CC BY-SA 4.0
|
||||
spring.datasource.hikari.auto-commit=false
|
||||
|
||||
logging.level.app.mealsmadeeasy.api=debug
|
||||
@ -29,6 +29,13 @@ public class RecipeDraft {
|
||||
private String rawText;
|
||||
}
|
||||
|
||||
@Data
|
||||
public static class IngredientDraft {
|
||||
private @Nullable String amount;
|
||||
private String name;
|
||||
private @Nullable String notes;
|
||||
}
|
||||
|
||||
@Id
|
||||
@GeneratedValue(strategy = GenerationType.UUID)
|
||||
@Column(nullable = false, unique = true, updatable = false)
|
||||
@ -48,6 +55,10 @@ public class RecipeDraft {
|
||||
private @Nullable Integer totalTime;
|
||||
private @Nullable String rawText;
|
||||
|
||||
@Type(JsonBinaryType.class)
|
||||
@Column(columnDefinition = "jsonb")
|
||||
private @Nullable List<IngredientDraft> ingredients;
|
||||
|
||||
@ManyToOne(optional = false)
|
||||
@JoinColumn(name = "owner_id", nullable = false)
|
||||
private User owner;
|
||||
|
||||
@ -6,14 +6,20 @@ import app.mealsmadeeasy.api.job.Job;
|
||||
import app.mealsmadeeasy.api.job.JobHandler;
|
||||
import app.mealsmadeeasy.api.recipe.RecipeDraft;
|
||||
import app.mealsmadeeasy.api.recipe.RecipeService;
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.chat.messages.Message;
|
||||
import org.springframework.ai.chat.messages.SystemMessage;
|
||||
import org.springframework.ai.chat.messages.UserMessage;
|
||||
import org.springframework.ai.chat.model.ChatModel;
|
||||
import org.springframework.ai.chat.model.ChatResponse;
|
||||
import org.springframework.ai.chat.prompt.ChatOptions;
|
||||
import org.springframework.ai.chat.prompt.Prompt;
|
||||
import org.springframework.ai.content.Media;
|
||||
import org.springframework.ai.converter.BeanOutputConverter;
|
||||
import org.springframework.ai.ollama.api.OllamaChatOptions;
|
||||
import org.springframework.core.io.InputStreamResource;
|
||||
import org.springframework.stereotype.Component;
|
||||
@ -23,16 +29,32 @@ import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class RecipeInferJobHandler implements JobHandler<RecipeInferJobPayload> {
|
||||
|
||||
private record RecipeExtraction(
|
||||
@JsonProperty(required = true) String title,
|
||||
@JsonProperty(required = true) List<IngredientExtraction> ingredients
|
||||
) {}
|
||||
|
||||
private record IngredientExtraction(
|
||||
String amount,
|
||||
@JsonProperty(required = true) String name,
|
||||
String notes
|
||||
) {}
|
||||
|
||||
public static final String JOB_KEY = "RECIPE_INFER_JOB";
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(RecipeInferJobHandler.class);
|
||||
|
||||
private final FileService fileService;
|
||||
private final RecipeService recipeService;
|
||||
private final ChatModel chatModel;
|
||||
private final BeanOutputConverter<RecipeExtraction> extractionConverter =
|
||||
new BeanOutputConverter<>(RecipeExtraction.class);
|
||||
|
||||
@Override
|
||||
public Class<RecipeInferJobPayload> getPayloadType() {
|
||||
@ -46,6 +68,7 @@ public class RecipeInferJobHandler implements JobHandler<RecipeInferJobPayload>
|
||||
|
||||
@Override
|
||||
public void handle(Job job, RecipeInferJobPayload payload) {
|
||||
logger.debug("Starting recipe inference job {}", job.getId());
|
||||
final File sourceFile = this.fileService.getById(payload.fileId());
|
||||
final InputStream sourceFileContent;
|
||||
try {
|
||||
@ -54,13 +77,14 @@ public class RecipeInferJobHandler implements JobHandler<RecipeInferJobPayload>
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
// 1. OCR of recipe image
|
||||
final Media sourceFileMedia = Media.builder()
|
||||
.data(new InputStreamResource(sourceFileContent))
|
||||
.mimeType(MimeType.valueOf(sourceFile.getMimeType()))
|
||||
.build();
|
||||
|
||||
final Message ocrMessage = UserMessage.builder()
|
||||
.text("Convert the recipe in the image to Markdown.")
|
||||
.text("Convert the recipe in the given file to Markdown.")
|
||||
.media(sourceFileMedia)
|
||||
.build();
|
||||
final Prompt ocrPrompt = Prompt.builder()
|
||||
@ -69,26 +93,103 @@ public class RecipeInferJobHandler implements JobHandler<RecipeInferJobPayload>
|
||||
.build();
|
||||
final ChatResponse ocrResponse = this.chatModel.call(ocrPrompt);
|
||||
final String fullMarkdownText = ocrResponse.getResult().getOutput().getText();
|
||||
if (fullMarkdownText == null) {
|
||||
throw new RuntimeException("fullMarkdownText from ocr came back null");
|
||||
}
|
||||
|
||||
// get recipe draft
|
||||
logger.debug("ocr returned:\n{}", fullMarkdownText);
|
||||
|
||||
// 2. Extract title and ingredients
|
||||
final Message extractSystemMessage = SystemMessage.builder()
|
||||
.text("""
|
||||
Extract from the given Markdown food recipe the title and ingredients.
|
||||
For each ingredient, you must provide the name; amount and notes are optional.
|
||||
|
||||
Here is an example of a full extraction:
|
||||
|
||||
```json
|
||||
{
|
||||
"title": "Delicious Recipe",
|
||||
"ingredients": [
|
||||
{
|
||||
"amount": "1 tablespoon",
|
||||
"name": "garlic powder"
|
||||
},
|
||||
{
|
||||
"amount": "1 lb",
|
||||
"name": "tofu",
|
||||
"notes": "pressed and dried"
|
||||
},
|
||||
{
|
||||
"name": "salt and pepper"
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
""".stripIndent().trim()
|
||||
)
|
||||
.build();
|
||||
final Message extractUserMessage = UserMessage.builder()
|
||||
.text(fullMarkdownText)
|
||||
.build();
|
||||
final ChatOptions extractChatOptions = OllamaChatOptions.builder()
|
||||
.model("gemma3:latest")
|
||||
.format(extractionConverter.getJsonSchemaMap())
|
||||
.build();
|
||||
final Prompt extractPrompt = Prompt.builder()
|
||||
.messages(extractSystemMessage, extractUserMessage)
|
||||
.chatOptions(extractChatOptions)
|
||||
.build();
|
||||
final ChatResponse extractResponse = this.chatModel.call(extractPrompt);
|
||||
final String extractContent = extractResponse.getResult().getOutput().getText();
|
||||
if (extractContent == null) {
|
||||
throw new RuntimeException("extractContent from extract came back null");
|
||||
}
|
||||
logger.debug("extract returned:\n{}", extractContent);
|
||||
|
||||
final RecipeExtraction extraction = this.extractionConverter.convert(extractContent);
|
||||
|
||||
// 3. Get draft and set props from extraction
|
||||
final @Nullable RecipeDraft recipeDraft = this.recipeService.getDraftById(payload.recipeDraftId());
|
||||
if (recipeDraft == null) {
|
||||
throw new RuntimeException("Recipe draft not found");
|
||||
}
|
||||
|
||||
// set props on draft from ai output, etc.
|
||||
recipeDraft.setTitle(extraction.title());
|
||||
|
||||
// todo: calculate slug
|
||||
|
||||
// ingredients
|
||||
if (recipeDraft.getIngredients() == null) {
|
||||
recipeDraft.setIngredients(new ArrayList<>());
|
||||
}
|
||||
recipeDraft.getIngredients().addAll(extraction.ingredients().stream()
|
||||
.map(extractedIngredient -> {
|
||||
final RecipeDraft.IngredientDraft ingredientDraft = new RecipeDraft.IngredientDraft();
|
||||
ingredientDraft.setAmount(extractedIngredient.amount());
|
||||
ingredientDraft.setName(extractedIngredient.name());
|
||||
ingredientDraft.setNotes(extractedIngredient.notes());
|
||||
return ingredientDraft;
|
||||
})
|
||||
.toList()
|
||||
);
|
||||
|
||||
// other props
|
||||
recipeDraft.setRawText(fullMarkdownText);
|
||||
recipeDraft.setState(RecipeDraft.State.ENTER_DATA);
|
||||
|
||||
// save inference for later
|
||||
if (recipeDraft.getInferences() == null) {
|
||||
recipeDraft.setInferences(new ArrayList<>());
|
||||
}
|
||||
final RecipeDraft.RecipeDraftInference inference = new RecipeDraft.RecipeDraftInference();
|
||||
inference.setTitle("TODO: inferred title");
|
||||
inference.setTitle(extraction.title());
|
||||
inference.setRawText(fullMarkdownText);
|
||||
inference.setInferredAt(OffsetDateTime.now());
|
||||
recipeDraft.getInferences().add(inference);
|
||||
|
||||
logger.debug("Recipe infer job completed, saving to db...");
|
||||
|
||||
this.recipeService.saveDraft(recipeDraft);
|
||||
}
|
||||
|
||||
|
||||
@ -0,0 +1 @@
|
||||
ALTER TABLE recipe_draft ADD COLUMN ingredients JSONB;
|
||||
Loading…
Reference in New Issue
Block a user