Add second AI call to recipe infer job.
This commit is contained in:
parent
b19dc42094
commit
547c04fbab
@ -75,6 +75,9 @@ public class RecipeInferJobIntegrationTests {
|
|||||||
|
|
||||||
final RecipeDraft draftWithInference = this.recipeService.getDraftById(draft.getId());
|
final RecipeDraft draftWithInference = this.recipeService.getDraftById(draft.getId());
|
||||||
|
|
||||||
|
assertThat(draftWithInference.getTitle(), is(notNullValue()));
|
||||||
|
assertThat(draftWithInference.getIngredients(), is(notNullValue()));
|
||||||
|
|
||||||
assertThat(draftWithInference.getInferences(), is(notNullValue()));
|
assertThat(draftWithInference.getInferences(), is(notNullValue()));
|
||||||
final List<RecipeDraft.RecipeDraftInference> inferences = draftWithInference.getInferences();
|
final List<RecipeDraft.RecipeDraftInference> inferences = draftWithInference.getInferences();
|
||||||
|
|
||||||
|
|||||||
@ -11,3 +11,5 @@ app.mealsmadeeasy.api.files.bucketName=files
|
|||||||
# Posted by Iogui, modified by community. See post 'Timeline' for change history
|
# Posted by Iogui, modified by community. See post 'Timeline' for change history
|
||||||
# Retrieved 2025-12-25, License - CC BY-SA 4.0
|
# Retrieved 2025-12-25, License - CC BY-SA 4.0
|
||||||
spring.datasource.hikari.auto-commit=false
|
spring.datasource.hikari.auto-commit=false
|
||||||
|
|
||||||
|
logging.level.app.mealsmadeeasy.api=debug
|
||||||
@ -29,6 +29,13 @@ public class RecipeDraft {
|
|||||||
private String rawText;
|
private String rawText;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Data
|
||||||
|
public static class IngredientDraft {
|
||||||
|
private @Nullable String amount;
|
||||||
|
private String name;
|
||||||
|
private @Nullable String notes;
|
||||||
|
}
|
||||||
|
|
||||||
@Id
|
@Id
|
||||||
@GeneratedValue(strategy = GenerationType.UUID)
|
@GeneratedValue(strategy = GenerationType.UUID)
|
||||||
@Column(nullable = false, unique = true, updatable = false)
|
@Column(nullable = false, unique = true, updatable = false)
|
||||||
@ -48,6 +55,10 @@ public class RecipeDraft {
|
|||||||
private @Nullable Integer totalTime;
|
private @Nullable Integer totalTime;
|
||||||
private @Nullable String rawText;
|
private @Nullable String rawText;
|
||||||
|
|
||||||
|
@Type(JsonBinaryType.class)
|
||||||
|
@Column(columnDefinition = "jsonb")
|
||||||
|
private @Nullable List<IngredientDraft> ingredients;
|
||||||
|
|
||||||
@ManyToOne(optional = false)
|
@ManyToOne(optional = false)
|
||||||
@JoinColumn(name = "owner_id", nullable = false)
|
@JoinColumn(name = "owner_id", nullable = false)
|
||||||
private User owner;
|
private User owner;
|
||||||
|
|||||||
@ -6,14 +6,20 @@ import app.mealsmadeeasy.api.job.Job;
|
|||||||
import app.mealsmadeeasy.api.job.JobHandler;
|
import app.mealsmadeeasy.api.job.JobHandler;
|
||||||
import app.mealsmadeeasy.api.recipe.RecipeDraft;
|
import app.mealsmadeeasy.api.recipe.RecipeDraft;
|
||||||
import app.mealsmadeeasy.api.recipe.RecipeService;
|
import app.mealsmadeeasy.api.recipe.RecipeService;
|
||||||
|
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||||
import lombok.RequiredArgsConstructor;
|
import lombok.RequiredArgsConstructor;
|
||||||
import org.jetbrains.annotations.Nullable;
|
import org.jetbrains.annotations.Nullable;
|
||||||
|
import org.slf4j.Logger;
|
||||||
|
import org.slf4j.LoggerFactory;
|
||||||
import org.springframework.ai.chat.messages.Message;
|
import org.springframework.ai.chat.messages.Message;
|
||||||
|
import org.springframework.ai.chat.messages.SystemMessage;
|
||||||
import org.springframework.ai.chat.messages.UserMessage;
|
import org.springframework.ai.chat.messages.UserMessage;
|
||||||
import org.springframework.ai.chat.model.ChatModel;
|
import org.springframework.ai.chat.model.ChatModel;
|
||||||
import org.springframework.ai.chat.model.ChatResponse;
|
import org.springframework.ai.chat.model.ChatResponse;
|
||||||
|
import org.springframework.ai.chat.prompt.ChatOptions;
|
||||||
import org.springframework.ai.chat.prompt.Prompt;
|
import org.springframework.ai.chat.prompt.Prompt;
|
||||||
import org.springframework.ai.content.Media;
|
import org.springframework.ai.content.Media;
|
||||||
|
import org.springframework.ai.converter.BeanOutputConverter;
|
||||||
import org.springframework.ai.ollama.api.OllamaChatOptions;
|
import org.springframework.ai.ollama.api.OllamaChatOptions;
|
||||||
import org.springframework.core.io.InputStreamResource;
|
import org.springframework.core.io.InputStreamResource;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
@ -23,16 +29,32 @@ import java.io.IOException;
|
|||||||
import java.io.InputStream;
|
import java.io.InputStream;
|
||||||
import java.time.OffsetDateTime;
|
import java.time.OffsetDateTime;
|
||||||
import java.util.ArrayList;
|
import java.util.ArrayList;
|
||||||
|
import java.util.List;
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
@RequiredArgsConstructor
|
@RequiredArgsConstructor
|
||||||
public class RecipeInferJobHandler implements JobHandler<RecipeInferJobPayload> {
|
public class RecipeInferJobHandler implements JobHandler<RecipeInferJobPayload> {
|
||||||
|
|
||||||
|
private record RecipeExtraction(
|
||||||
|
@JsonProperty(required = true) String title,
|
||||||
|
@JsonProperty(required = true) List<IngredientExtraction> ingredients
|
||||||
|
) {}
|
||||||
|
|
||||||
|
private record IngredientExtraction(
|
||||||
|
String amount,
|
||||||
|
@JsonProperty(required = true) String name,
|
||||||
|
String notes
|
||||||
|
) {}
|
||||||
|
|
||||||
public static final String JOB_KEY = "RECIPE_INFER_JOB";
|
public static final String JOB_KEY = "RECIPE_INFER_JOB";
|
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(RecipeInferJobHandler.class);
|
||||||
|
|
||||||
private final FileService fileService;
|
private final FileService fileService;
|
||||||
private final RecipeService recipeService;
|
private final RecipeService recipeService;
|
||||||
private final ChatModel chatModel;
|
private final ChatModel chatModel;
|
||||||
|
private final BeanOutputConverter<RecipeExtraction> extractionConverter =
|
||||||
|
new BeanOutputConverter<>(RecipeExtraction.class);
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public Class<RecipeInferJobPayload> getPayloadType() {
|
public Class<RecipeInferJobPayload> getPayloadType() {
|
||||||
@ -46,6 +68,7 @@ public class RecipeInferJobHandler implements JobHandler<RecipeInferJobPayload>
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void handle(Job job, RecipeInferJobPayload payload) {
|
public void handle(Job job, RecipeInferJobPayload payload) {
|
||||||
|
logger.debug("Starting recipe inference job {}", job.getId());
|
||||||
final File sourceFile = this.fileService.getById(payload.fileId());
|
final File sourceFile = this.fileService.getById(payload.fileId());
|
||||||
final InputStream sourceFileContent;
|
final InputStream sourceFileContent;
|
||||||
try {
|
try {
|
||||||
@ -54,13 +77,14 @@ public class RecipeInferJobHandler implements JobHandler<RecipeInferJobPayload>
|
|||||||
throw new RuntimeException(e);
|
throw new RuntimeException(e);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// 1. OCR of recipe image
|
||||||
final Media sourceFileMedia = Media.builder()
|
final Media sourceFileMedia = Media.builder()
|
||||||
.data(new InputStreamResource(sourceFileContent))
|
.data(new InputStreamResource(sourceFileContent))
|
||||||
.mimeType(MimeType.valueOf(sourceFile.getMimeType()))
|
.mimeType(MimeType.valueOf(sourceFile.getMimeType()))
|
||||||
.build();
|
.build();
|
||||||
|
|
||||||
final Message ocrMessage = UserMessage.builder()
|
final Message ocrMessage = UserMessage.builder()
|
||||||
.text("Convert the recipe in the image to Markdown.")
|
.text("Convert the recipe in the given file to Markdown.")
|
||||||
.media(sourceFileMedia)
|
.media(sourceFileMedia)
|
||||||
.build();
|
.build();
|
||||||
final Prompt ocrPrompt = Prompt.builder()
|
final Prompt ocrPrompt = Prompt.builder()
|
||||||
@ -69,26 +93,103 @@ public class RecipeInferJobHandler implements JobHandler<RecipeInferJobPayload>
|
|||||||
.build();
|
.build();
|
||||||
final ChatResponse ocrResponse = this.chatModel.call(ocrPrompt);
|
final ChatResponse ocrResponse = this.chatModel.call(ocrPrompt);
|
||||||
final String fullMarkdownText = ocrResponse.getResult().getOutput().getText();
|
final String fullMarkdownText = ocrResponse.getResult().getOutput().getText();
|
||||||
|
if (fullMarkdownText == null) {
|
||||||
|
throw new RuntimeException("fullMarkdownText from ocr came back null");
|
||||||
|
}
|
||||||
|
|
||||||
// get recipe draft
|
logger.debug("ocr returned:\n{}", fullMarkdownText);
|
||||||
|
|
||||||
|
// 2. Extract title and ingredients
|
||||||
|
final Message extractSystemMessage = SystemMessage.builder()
|
||||||
|
.text("""
|
||||||
|
Extract from the given Markdown food recipe the title and ingredients.
|
||||||
|
For each ingredient, you must provide the name; amount and notes are optional.
|
||||||
|
|
||||||
|
Here is an example of a full extraction:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"title": "Delicious Recipe",
|
||||||
|
"ingredients": [
|
||||||
|
{
|
||||||
|
"amount": "1 tablespoon",
|
||||||
|
"name": "garlic powder"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"amount": "1 lb",
|
||||||
|
"name": "tofu",
|
||||||
|
"notes": "pressed and dried"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "salt and pepper"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
""".stripIndent().trim()
|
||||||
|
)
|
||||||
|
.build();
|
||||||
|
final Message extractUserMessage = UserMessage.builder()
|
||||||
|
.text(fullMarkdownText)
|
||||||
|
.build();
|
||||||
|
final ChatOptions extractChatOptions = OllamaChatOptions.builder()
|
||||||
|
.model("gemma3:latest")
|
||||||
|
.format(extractionConverter.getJsonSchemaMap())
|
||||||
|
.build();
|
||||||
|
final Prompt extractPrompt = Prompt.builder()
|
||||||
|
.messages(extractSystemMessage, extractUserMessage)
|
||||||
|
.chatOptions(extractChatOptions)
|
||||||
|
.build();
|
||||||
|
final ChatResponse extractResponse = this.chatModel.call(extractPrompt);
|
||||||
|
final String extractContent = extractResponse.getResult().getOutput().getText();
|
||||||
|
if (extractContent == null) {
|
||||||
|
throw new RuntimeException("extractContent from extract came back null");
|
||||||
|
}
|
||||||
|
logger.debug("extract returned:\n{}", extractContent);
|
||||||
|
|
||||||
|
final RecipeExtraction extraction = this.extractionConverter.convert(extractContent);
|
||||||
|
|
||||||
|
// 3. Get draft and set props from extraction
|
||||||
final @Nullable RecipeDraft recipeDraft = this.recipeService.getDraftById(payload.recipeDraftId());
|
final @Nullable RecipeDraft recipeDraft = this.recipeService.getDraftById(payload.recipeDraftId());
|
||||||
if (recipeDraft == null) {
|
if (recipeDraft == null) {
|
||||||
throw new RuntimeException("Recipe draft not found");
|
throw new RuntimeException("Recipe draft not found");
|
||||||
}
|
}
|
||||||
|
|
||||||
// set props on draft from ai output, etc.
|
recipeDraft.setTitle(extraction.title());
|
||||||
|
|
||||||
|
// todo: calculate slug
|
||||||
|
|
||||||
|
// ingredients
|
||||||
|
if (recipeDraft.getIngredients() == null) {
|
||||||
|
recipeDraft.setIngredients(new ArrayList<>());
|
||||||
|
}
|
||||||
|
recipeDraft.getIngredients().addAll(extraction.ingredients().stream()
|
||||||
|
.map(extractedIngredient -> {
|
||||||
|
final RecipeDraft.IngredientDraft ingredientDraft = new RecipeDraft.IngredientDraft();
|
||||||
|
ingredientDraft.setAmount(extractedIngredient.amount());
|
||||||
|
ingredientDraft.setName(extractedIngredient.name());
|
||||||
|
ingredientDraft.setNotes(extractedIngredient.notes());
|
||||||
|
return ingredientDraft;
|
||||||
|
})
|
||||||
|
.toList()
|
||||||
|
);
|
||||||
|
|
||||||
|
// other props
|
||||||
recipeDraft.setRawText(fullMarkdownText);
|
recipeDraft.setRawText(fullMarkdownText);
|
||||||
recipeDraft.setState(RecipeDraft.State.ENTER_DATA);
|
recipeDraft.setState(RecipeDraft.State.ENTER_DATA);
|
||||||
|
|
||||||
|
// save inference for later
|
||||||
if (recipeDraft.getInferences() == null) {
|
if (recipeDraft.getInferences() == null) {
|
||||||
recipeDraft.setInferences(new ArrayList<>());
|
recipeDraft.setInferences(new ArrayList<>());
|
||||||
}
|
}
|
||||||
final RecipeDraft.RecipeDraftInference inference = new RecipeDraft.RecipeDraftInference();
|
final RecipeDraft.RecipeDraftInference inference = new RecipeDraft.RecipeDraftInference();
|
||||||
inference.setTitle("TODO: inferred title");
|
inference.setTitle(extraction.title());
|
||||||
inference.setRawText(fullMarkdownText);
|
inference.setRawText(fullMarkdownText);
|
||||||
inference.setInferredAt(OffsetDateTime.now());
|
inference.setInferredAt(OffsetDateTime.now());
|
||||||
recipeDraft.getInferences().add(inference);
|
recipeDraft.getInferences().add(inference);
|
||||||
|
|
||||||
|
logger.debug("Recipe infer job completed, saving to db...");
|
||||||
|
|
||||||
this.recipeService.saveDraft(recipeDraft);
|
this.recipeService.saveDraft(recipeDraft);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -0,0 +1 @@
|
|||||||
|
ALTER TABLE recipe_draft ADD COLUMN ingredients JSONB;
|
||||||
Loading…
Reference in New Issue
Block a user