Compare commits
No commits in common. "24832449f6e2546a61e1c9db3ce1a210350f34c2" and "77b94e69880e4da241ce9cddcbbd6d6831c8d57c" have entirely different histories.
24832449f6
...
77b94e6988
1
.gitignore
vendored
1
.gitignore
vendored
@ -38,4 +38,3 @@ out/
|
||||
|
||||
# Custom
|
||||
.env
|
||||
/pgadmin_data/
|
||||
|
||||
@ -32,17 +32,6 @@ services:
|
||||
- /data
|
||||
- --console-address
|
||||
- :9001
|
||||
pgadmin:
|
||||
image: dpage/pgadmin4:latest
|
||||
container_name: pgadmin
|
||||
environment:
|
||||
PGADMIN_DEFAULT_EMAIL: admin@admin.com # Your login email
|
||||
PGADMIN_DEFAULT_PASSWORD: root # Your login password
|
||||
ports:
|
||||
- "5555:80"
|
||||
volumes:
|
||||
- ./pgadmin_data:/var/lib/pgadmin # Persistent storage for pgAdmin settings
|
||||
restart: always
|
||||
volumes:
|
||||
postgres-data:
|
||||
minio-data:
|
||||
|
||||
@ -1,31 +1,56 @@
|
||||
package app.mealsmadeeasy.api;
|
||||
|
||||
import app.mealsmadeeasy.api.job.JobService;
|
||||
import app.mealsmadeeasy.api.recipe.Recipe;
|
||||
import app.mealsmadeeasy.api.recipe.RecipeEmbedding;
|
||||
import app.mealsmadeeasy.api.recipe.RecipeRepository;
|
||||
import app.mealsmadeeasy.api.recipe.job.RecipeEmbeddingJobHandler;
|
||||
import app.mealsmadeeasy.api.recipe.job.RecipeEmbeddingJobPayload;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import app.mealsmadeeasy.api.recipe.RecipeService;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.boot.ApplicationArguments;
|
||||
import org.springframework.boot.ApplicationRunner;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
import java.util.List;
|
||||
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
@ConditionalOnProperty(name = "backfill.recipe-embeddings.enabled", havingValue = "true")
|
||||
public class BackfillRecipeEmbeddings implements ApplicationRunner {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(BackfillRecipeEmbeddings.class);
|
||||
|
||||
private final RecipeRepository recipeRepository;
|
||||
private final JobService jobService;
|
||||
private final RecipeService recipeService;
|
||||
private final EmbeddingModel embeddingModel;
|
||||
|
||||
public BackfillRecipeEmbeddings(
|
||||
RecipeRepository recipeRepository,
|
||||
RecipeService recipeService,
|
||||
EmbeddingModel embeddingModel
|
||||
) {
|
||||
this.recipeRepository = recipeRepository;
|
||||
this.recipeService = recipeService;
|
||||
this.embeddingModel = embeddingModel;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(ApplicationArguments args) {
|
||||
final List<Recipe> recipeEntities = this.recipeRepository.findAllByEmbeddingIsNull();
|
||||
for (final Recipe recipe : recipeEntities) {
|
||||
this.jobService.create(RecipeEmbeddingJobHandler.JOB_KEY, new RecipeEmbeddingJobPayload(recipe.getId()));
|
||||
logger.info("Calculating embedding for {}", recipe);
|
||||
final String renderedMarkdown = this.recipeService.getRenderedMarkdown(recipe);
|
||||
final String toEmbed = "<h1>" + recipe.getTitle() + "</h1>" + renderedMarkdown;
|
||||
final float[] embedding = this.embeddingModel.embed(toEmbed);
|
||||
|
||||
final RecipeEmbedding recipeEmbedding = new RecipeEmbedding();
|
||||
recipeEmbedding.setRecipe(recipe);
|
||||
recipeEmbedding.setEmbedding(embedding);
|
||||
recipeEmbedding.setTimestamp(OffsetDateTime.now());
|
||||
recipe.setEmbedding(recipeEmbedding);
|
||||
|
||||
this.recipeRepository.save(recipe);
|
||||
}
|
||||
this.recipeRepository.flush();
|
||||
}
|
||||
|
||||
@ -4,8 +4,6 @@ import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import jakarta.transaction.Transactional;
|
||||
import org.jetbrains.annotations.Nullable;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.context.ApplicationContext;
|
||||
import org.springframework.scheduling.annotation.Scheduled;
|
||||
import org.springframework.stereotype.Service;
|
||||
@ -21,8 +19,6 @@ import java.util.concurrent.ThreadLocalRandom;
|
||||
@Service
|
||||
public class JobService {
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(JobService.class);
|
||||
|
||||
private final ApplicationContext applicationContext;
|
||||
private final JobRepository jobRepository;
|
||||
private final ObjectMapper objectMapper;
|
||||
@ -87,7 +83,6 @@ public class JobService {
|
||||
job.setLockedAt(null);
|
||||
this.jobRepository.save(job);
|
||||
} catch (Exception e) {
|
||||
logger.error("Job {} {} threw an exception: {}", job.getId(), job.getJobKey(), e.getMessage());
|
||||
final int attemptCount = job.getAttempts() + 1;
|
||||
final boolean isDead = attemptCount >= job.getMaxAttempts();
|
||||
final OffsetDateTime runAfter = isDead
|
||||
|
||||
@ -6,8 +6,6 @@ import app.mealsmadeeasy.api.image.ImageService;
|
||||
import app.mealsmadeeasy.api.job.JobService;
|
||||
import app.mealsmadeeasy.api.markdown.MarkdownService;
|
||||
import app.mealsmadeeasy.api.recipe.body.RecipeAiSearchBody;
|
||||
import app.mealsmadeeasy.api.recipe.job.RecipeEmbeddingJobHandler;
|
||||
import app.mealsmadeeasy.api.recipe.job.RecipeEmbeddingJobPayload;
|
||||
import app.mealsmadeeasy.api.recipe.job.RecipeInferJobHandler;
|
||||
import app.mealsmadeeasy.api.recipe.job.RecipeInferJobPayload;
|
||||
import app.mealsmadeeasy.api.recipe.spec.RecipeCreateSpec;
|
||||
@ -54,9 +52,7 @@ public class RecipeService {
|
||||
draft.setRawText(spec.getRawText());
|
||||
draft.setMainImage(spec.getMainImage());
|
||||
draft.setIsPublic(spec.isPublic());
|
||||
final Recipe saved = this.recipeRepository.save(draft);
|
||||
this.jobService.create(RecipeEmbeddingJobHandler.JOB_KEY, new RecipeEmbeddingJobPayload(saved.getId()));
|
||||
return saved;
|
||||
return this.recipeRepository.save(draft);
|
||||
}
|
||||
|
||||
private Recipe getById(Integer id) {
|
||||
|
||||
@ -1,58 +0,0 @@
|
||||
package app.mealsmadeeasy.api.recipe.job;
|
||||
|
||||
import app.mealsmadeeasy.api.job.Job;
|
||||
import app.mealsmadeeasy.api.job.JobHandler;
|
||||
import app.mealsmadeeasy.api.recipe.Recipe;
|
||||
import app.mealsmadeeasy.api.recipe.RecipeEmbedding;
|
||||
import app.mealsmadeeasy.api.recipe.RecipeRepository;
|
||||
import app.mealsmadeeasy.api.recipe.RecipeService;
|
||||
import app.mealsmadeeasy.api.util.NoSuchEntityWithIdException;
|
||||
import lombok.RequiredArgsConstructor;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.ai.embedding.EmbeddingModel;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.time.OffsetDateTime;
|
||||
|
||||
@Component
|
||||
@RequiredArgsConstructor
|
||||
public class RecipeEmbeddingJobHandler implements JobHandler<RecipeEmbeddingJobPayload> {
|
||||
|
||||
public static final String JOB_KEY = "RECIPE_EMBEDDING";
|
||||
|
||||
private static final Logger logger = LoggerFactory.getLogger(RecipeEmbeddingJobHandler.class);
|
||||
|
||||
private final RecipeRepository recipeRepository;
|
||||
private final RecipeService recipeService;
|
||||
private final EmbeddingModel embeddingModel;
|
||||
|
||||
@Override
|
||||
public Class<RecipeEmbeddingJobPayload> getPayloadType() {
|
||||
return RecipeEmbeddingJobPayload.class;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getJobKey() {
|
||||
return JOB_KEY;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void handle(Job job, RecipeEmbeddingJobPayload payload) {
|
||||
logger.info("Calculating embedding for recipeId {}", payload.recipeId());
|
||||
final Recipe recipe = this.recipeRepository.findById(payload.recipeId())
|
||||
.orElseThrow(() -> new NoSuchEntityWithIdException(Recipe.class, payload.recipeId()));
|
||||
final String renderedMarkdown = this.recipeService.getRenderedMarkdown(recipe);
|
||||
final String toEmbed = "<h1>" + recipe.getTitle() + "</h1>\n" + renderedMarkdown;
|
||||
final float[] embedding = this.embeddingModel.embed(toEmbed);
|
||||
|
||||
final RecipeEmbedding recipeEmbedding = new RecipeEmbedding();
|
||||
recipeEmbedding.setRecipe(recipe);
|
||||
recipeEmbedding.setEmbedding(embedding);
|
||||
recipeEmbedding.setTimestamp(OffsetDateTime.now());
|
||||
recipe.setEmbedding(recipeEmbedding);
|
||||
this.recipeRepository.save(recipe);
|
||||
logger.info("Finished calculating embedding for recipeId {}", payload.recipeId());
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,3 +0,0 @@
|
||||
package app.mealsmadeeasy.api.recipe.job;
|
||||
|
||||
public record RecipeEmbeddingJobPayload(Integer recipeId) {}
|
||||
@ -20,4 +20,5 @@ app.mealsmadeeasy.api.files.bucketName=files
|
||||
|
||||
# AI
|
||||
spring.ai.vectorstore.pgvector.dimensions=1024
|
||||
spring.ai.ollama.chat.options.model=deepseek-ocr:latest
|
||||
spring.ai.ollama.init.pull-model-strategy=never
|
||||
Loading…
Reference in New Issue
Block a user