Compare commits
No commits in common. "24832449f6e2546a61e1c9db3ce1a210350f34c2" and "77b94e69880e4da241ce9cddcbbd6d6831c8d57c" have entirely different histories.
24832449f6
...
77b94e6988
1
.gitignore
vendored
1
.gitignore
vendored
@ -38,4 +38,3 @@ out/
|
|||||||
|
|
||||||
# Custom
|
# Custom
|
||||||
.env
|
.env
|
||||||
/pgadmin_data/
|
|
||||||
|
|||||||
@ -32,17 +32,6 @@ services:
|
|||||||
- /data
|
- /data
|
||||||
- --console-address
|
- --console-address
|
||||||
- :9001
|
- :9001
|
||||||
pgadmin:
|
|
||||||
image: dpage/pgadmin4:latest
|
|
||||||
container_name: pgadmin
|
|
||||||
environment:
|
|
||||||
PGADMIN_DEFAULT_EMAIL: admin@admin.com # Your login email
|
|
||||||
PGADMIN_DEFAULT_PASSWORD: root # Your login password
|
|
||||||
ports:
|
|
||||||
- "5555:80"
|
|
||||||
volumes:
|
|
||||||
- ./pgadmin_data:/var/lib/pgadmin # Persistent storage for pgAdmin settings
|
|
||||||
restart: always
|
|
||||||
volumes:
|
volumes:
|
||||||
postgres-data:
|
postgres-data:
|
||||||
minio-data:
|
minio-data:
|
||||||
|
|||||||
@ -1,31 +1,56 @@
|
|||||||
package app.mealsmadeeasy.api;
|
package app.mealsmadeeasy.api;
|
||||||
|
|
||||||
import app.mealsmadeeasy.api.job.JobService;
|
|
||||||
import app.mealsmadeeasy.api.recipe.Recipe;
|
import app.mealsmadeeasy.api.recipe.Recipe;
|
||||||
|
import app.mealsmadeeasy.api.recipe.RecipeEmbedding;
|
||||||
import app.mealsmadeeasy.api.recipe.RecipeRepository;
|
import app.mealsmadeeasy.api.recipe.RecipeRepository;
|
||||||
import app.mealsmadeeasy.api.recipe.job.RecipeEmbeddingJobHandler;
|
import app.mealsmadeeasy.api.recipe.RecipeService;
|
||||||
import app.mealsmadeeasy.api.recipe.job.RecipeEmbeddingJobPayload;
|
import org.slf4j.Logger;
|
||||||
import lombok.RequiredArgsConstructor;
|
import org.slf4j.LoggerFactory;
|
||||||
|
import org.springframework.ai.embedding.EmbeddingModel;
|
||||||
import org.springframework.boot.ApplicationArguments;
|
import org.springframework.boot.ApplicationArguments;
|
||||||
import org.springframework.boot.ApplicationRunner;
|
import org.springframework.boot.ApplicationRunner;
|
||||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||||
import org.springframework.stereotype.Component;
|
import org.springframework.stereotype.Component;
|
||||||
|
|
||||||
|
import java.time.OffsetDateTime;
|
||||||
import java.util.List;
|
import java.util.List;
|
||||||
|
|
||||||
@Component
|
@Component
|
||||||
@RequiredArgsConstructor
|
|
||||||
@ConditionalOnProperty(name = "backfill.recipe-embeddings.enabled", havingValue = "true")
|
@ConditionalOnProperty(name = "backfill.recipe-embeddings.enabled", havingValue = "true")
|
||||||
public class BackfillRecipeEmbeddings implements ApplicationRunner {
|
public class BackfillRecipeEmbeddings implements ApplicationRunner {
|
||||||
|
|
||||||
|
private static final Logger logger = LoggerFactory.getLogger(BackfillRecipeEmbeddings.class);
|
||||||
|
|
||||||
private final RecipeRepository recipeRepository;
|
private final RecipeRepository recipeRepository;
|
||||||
private final JobService jobService;
|
private final RecipeService recipeService;
|
||||||
|
private final EmbeddingModel embeddingModel;
|
||||||
|
|
||||||
|
public BackfillRecipeEmbeddings(
|
||||||
|
RecipeRepository recipeRepository,
|
||||||
|
RecipeService recipeService,
|
||||||
|
EmbeddingModel embeddingModel
|
||||||
|
) {
|
||||||
|
this.recipeRepository = recipeRepository;
|
||||||
|
this.recipeService = recipeService;
|
||||||
|
this.embeddingModel = embeddingModel;
|
||||||
|
}
|
||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void run(ApplicationArguments args) {
|
public void run(ApplicationArguments args) {
|
||||||
final List<Recipe> recipeEntities = this.recipeRepository.findAllByEmbeddingIsNull();
|
final List<Recipe> recipeEntities = this.recipeRepository.findAllByEmbeddingIsNull();
|
||||||
for (final Recipe recipe : recipeEntities) {
|
for (final Recipe recipe : recipeEntities) {
|
||||||
this.jobService.create(RecipeEmbeddingJobHandler.JOB_KEY, new RecipeEmbeddingJobPayload(recipe.getId()));
|
logger.info("Calculating embedding for {}", recipe);
|
||||||
|
final String renderedMarkdown = this.recipeService.getRenderedMarkdown(recipe);
|
||||||
|
final String toEmbed = "<h1>" + recipe.getTitle() + "</h1>" + renderedMarkdown;
|
||||||
|
final float[] embedding = this.embeddingModel.embed(toEmbed);
|
||||||
|
|
||||||
|
final RecipeEmbedding recipeEmbedding = new RecipeEmbedding();
|
||||||
|
recipeEmbedding.setRecipe(recipe);
|
||||||
|
recipeEmbedding.setEmbedding(embedding);
|
||||||
|
recipeEmbedding.setTimestamp(OffsetDateTime.now());
|
||||||
|
recipe.setEmbedding(recipeEmbedding);
|
||||||
|
|
||||||
|
this.recipeRepository.save(recipe);
|
||||||
}
|
}
|
||||||
this.recipeRepository.flush();
|
this.recipeRepository.flush();
|
||||||
}
|
}
|
||||||
|
|||||||
@ -4,8 +4,6 @@ import com.fasterxml.jackson.databind.JsonNode;
|
|||||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||||
import jakarta.transaction.Transactional;
|
import jakarta.transaction.Transactional;
|
||||||
import org.jetbrains.annotations.Nullable;
|
import org.jetbrains.annotations.Nullable;
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.context.ApplicationContext;
|
import org.springframework.context.ApplicationContext;
|
||||||
import org.springframework.scheduling.annotation.Scheduled;
|
import org.springframework.scheduling.annotation.Scheduled;
|
||||||
import org.springframework.stereotype.Service;
|
import org.springframework.stereotype.Service;
|
||||||
@ -21,8 +19,6 @@ import java.util.concurrent.ThreadLocalRandom;
|
|||||||
@Service
|
@Service
|
||||||
public class JobService {
|
public class JobService {
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(JobService.class);
|
|
||||||
|
|
||||||
private final ApplicationContext applicationContext;
|
private final ApplicationContext applicationContext;
|
||||||
private final JobRepository jobRepository;
|
private final JobRepository jobRepository;
|
||||||
private final ObjectMapper objectMapper;
|
private final ObjectMapper objectMapper;
|
||||||
@ -87,7 +83,6 @@ public class JobService {
|
|||||||
job.setLockedAt(null);
|
job.setLockedAt(null);
|
||||||
this.jobRepository.save(job);
|
this.jobRepository.save(job);
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
logger.error("Job {} {} threw an exception: {}", job.getId(), job.getJobKey(), e.getMessage());
|
|
||||||
final int attemptCount = job.getAttempts() + 1;
|
final int attemptCount = job.getAttempts() + 1;
|
||||||
final boolean isDead = attemptCount >= job.getMaxAttempts();
|
final boolean isDead = attemptCount >= job.getMaxAttempts();
|
||||||
final OffsetDateTime runAfter = isDead
|
final OffsetDateTime runAfter = isDead
|
||||||
|
|||||||
@ -6,8 +6,6 @@ import app.mealsmadeeasy.api.image.ImageService;
|
|||||||
import app.mealsmadeeasy.api.job.JobService;
|
import app.mealsmadeeasy.api.job.JobService;
|
||||||
import app.mealsmadeeasy.api.markdown.MarkdownService;
|
import app.mealsmadeeasy.api.markdown.MarkdownService;
|
||||||
import app.mealsmadeeasy.api.recipe.body.RecipeAiSearchBody;
|
import app.mealsmadeeasy.api.recipe.body.RecipeAiSearchBody;
|
||||||
import app.mealsmadeeasy.api.recipe.job.RecipeEmbeddingJobHandler;
|
|
||||||
import app.mealsmadeeasy.api.recipe.job.RecipeEmbeddingJobPayload;
|
|
||||||
import app.mealsmadeeasy.api.recipe.job.RecipeInferJobHandler;
|
import app.mealsmadeeasy.api.recipe.job.RecipeInferJobHandler;
|
||||||
import app.mealsmadeeasy.api.recipe.job.RecipeInferJobPayload;
|
import app.mealsmadeeasy.api.recipe.job.RecipeInferJobPayload;
|
||||||
import app.mealsmadeeasy.api.recipe.spec.RecipeCreateSpec;
|
import app.mealsmadeeasy.api.recipe.spec.RecipeCreateSpec;
|
||||||
@ -54,9 +52,7 @@ public class RecipeService {
|
|||||||
draft.setRawText(spec.getRawText());
|
draft.setRawText(spec.getRawText());
|
||||||
draft.setMainImage(spec.getMainImage());
|
draft.setMainImage(spec.getMainImage());
|
||||||
draft.setIsPublic(spec.isPublic());
|
draft.setIsPublic(spec.isPublic());
|
||||||
final Recipe saved = this.recipeRepository.save(draft);
|
return this.recipeRepository.save(draft);
|
||||||
this.jobService.create(RecipeEmbeddingJobHandler.JOB_KEY, new RecipeEmbeddingJobPayload(saved.getId()));
|
|
||||||
return saved;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private Recipe getById(Integer id) {
|
private Recipe getById(Integer id) {
|
||||||
|
|||||||
@ -1,58 +0,0 @@
|
|||||||
package app.mealsmadeeasy.api.recipe.job;
|
|
||||||
|
|
||||||
import app.mealsmadeeasy.api.job.Job;
|
|
||||||
import app.mealsmadeeasy.api.job.JobHandler;
|
|
||||||
import app.mealsmadeeasy.api.recipe.Recipe;
|
|
||||||
import app.mealsmadeeasy.api.recipe.RecipeEmbedding;
|
|
||||||
import app.mealsmadeeasy.api.recipe.RecipeRepository;
|
|
||||||
import app.mealsmadeeasy.api.recipe.RecipeService;
|
|
||||||
import app.mealsmadeeasy.api.util.NoSuchEntityWithIdException;
|
|
||||||
import lombok.RequiredArgsConstructor;
|
|
||||||
import org.slf4j.Logger;
|
|
||||||
import org.slf4j.LoggerFactory;
|
|
||||||
import org.springframework.ai.embedding.EmbeddingModel;
|
|
||||||
import org.springframework.stereotype.Component;
|
|
||||||
|
|
||||||
import java.time.OffsetDateTime;
|
|
||||||
|
|
||||||
@Component
|
|
||||||
@RequiredArgsConstructor
|
|
||||||
public class RecipeEmbeddingJobHandler implements JobHandler<RecipeEmbeddingJobPayload> {
|
|
||||||
|
|
||||||
public static final String JOB_KEY = "RECIPE_EMBEDDING";
|
|
||||||
|
|
||||||
private static final Logger logger = LoggerFactory.getLogger(RecipeEmbeddingJobHandler.class);
|
|
||||||
|
|
||||||
private final RecipeRepository recipeRepository;
|
|
||||||
private final RecipeService recipeService;
|
|
||||||
private final EmbeddingModel embeddingModel;
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public Class<RecipeEmbeddingJobPayload> getPayloadType() {
|
|
||||||
return RecipeEmbeddingJobPayload.class;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public String getJobKey() {
|
|
||||||
return JOB_KEY;
|
|
||||||
}
|
|
||||||
|
|
||||||
@Override
|
|
||||||
public void handle(Job job, RecipeEmbeddingJobPayload payload) {
|
|
||||||
logger.info("Calculating embedding for recipeId {}", payload.recipeId());
|
|
||||||
final Recipe recipe = this.recipeRepository.findById(payload.recipeId())
|
|
||||||
.orElseThrow(() -> new NoSuchEntityWithIdException(Recipe.class, payload.recipeId()));
|
|
||||||
final String renderedMarkdown = this.recipeService.getRenderedMarkdown(recipe);
|
|
||||||
final String toEmbed = "<h1>" + recipe.getTitle() + "</h1>\n" + renderedMarkdown;
|
|
||||||
final float[] embedding = this.embeddingModel.embed(toEmbed);
|
|
||||||
|
|
||||||
final RecipeEmbedding recipeEmbedding = new RecipeEmbedding();
|
|
||||||
recipeEmbedding.setRecipe(recipe);
|
|
||||||
recipeEmbedding.setEmbedding(embedding);
|
|
||||||
recipeEmbedding.setTimestamp(OffsetDateTime.now());
|
|
||||||
recipe.setEmbedding(recipeEmbedding);
|
|
||||||
this.recipeRepository.save(recipe);
|
|
||||||
logger.info("Finished calculating embedding for recipeId {}", payload.recipeId());
|
|
||||||
}
|
|
||||||
|
|
||||||
}
|
|
||||||
@ -1,3 +0,0 @@
|
|||||||
package app.mealsmadeeasy.api.recipe.job;
|
|
||||||
|
|
||||||
public record RecipeEmbeddingJobPayload(Integer recipeId) {}
|
|
||||||
@ -20,4 +20,5 @@ app.mealsmadeeasy.api.files.bucketName=files
|
|||||||
|
|
||||||
# AI
|
# AI
|
||||||
spring.ai.vectorstore.pgvector.dimensions=1024
|
spring.ai.vectorstore.pgvector.dimensions=1024
|
||||||
|
spring.ai.ollama.chat.options.model=deepseek-ocr:latest
|
||||||
spring.ai.ollama.init.pull-model-strategy=never
|
spring.ai.ollama.init.pull-model-strategy=never
|
||||||
Loading…
Reference in New Issue
Block a user