// movie-seed.service.ts
import { Injectable } from '@nestjs/common';  
import { InjectRepository } from '@nestjs/typeorm';  

import { Repository } from 'typeorm';  
import { MovieEntity } from '../../../../movies/infrastructure/persistence/relational/entities/movie.entity';  

import { createReadStream } from 'fs';  
import { parse } from 'csv-parse';  
import path from 'path';  

@Injectable()  
export class MovieSeedService {  
  constructor(  
    @InjectRepository(MovieEntity)  
    private repository: Repository,  
  ) {}  

  async run() {  
    const countMovies = await this.repository.count({});  

    if (!countMovies) {  
      const parser = createReadStream(  
        path.join(__dirname, 'movie_titles_and_script.csv'),  
      ).pipe(parse({ delimiter: ',', from_line: 2 }));  

      /* In a real application, you might consider using a bulk command, like PostgreSQL's  
         COPY (https://www.postgresql.org/docs/current/sql-copy.html)       
     */      
     for await (const data of parser) {  
        await this.saveCsvRow(  
          this.repository,  
          ...(data as [string, string, string, string, string, string, string]),  
        );  
      }  
    }  
  }  

  async revert() {  
    await this.repository.clear();  
  }  

  /* eslint-disable @typescript-eslint/no-unused-vars */  
  async saveCsvRow(  
    repository: Repository,  
    id: string,  
    title: string,  
    year: string,  
    rating: string,  
    numVotes: string,  
    categoriesUnprocessed: string,  
    script,  
  ): Promise {  
    await repository.save(  
      repository.create({  
        title: title,  
        script: script,  
      }),  
    );  
  }  
}

npm run seed:run:relational
npm run seed:run:search

services:  
  ...
  # Apache License v2.0, Oct 01, 2024  
  # Courtesy of https://opensearch.org/downloads.html  
  opensearch:  
    image: opensearchproject/opensearch:2.17.1  
    environment:  
      - discovery.type=single-node  
      - plugins.security.disabled=true  
      - http.host=0.0.0.0  
      - transport.host=127.0.0.1  
      - OPENSEARCH_INITIAL_ADMIN_PASSWORD=passRT%^#234  
      - DISABLE_SECURITY_PLUGIN=true  
    ulimits:  
      memlock:  
        soft: -1  
        hard: -1  
      nofile:  
        soft: 65536
        hard: 65536  
    volumes:  
      - opensearch-data:/usr/share/opensearch/data  
    ports:  
      - "9200:9200"  
  # End Apache License v2.0  

 ...

...
volumes:  
  boilerplate-db:
  opensearch-data:

-ELASTICSEARCH_USERNAME=elastic
-ELASTICSEARCH_PASSWORD=
+ELASTICSEARCH_USERNAME=admin
+ELASTICSEARCH_PASSWORD=passRT%^#234

// movies-search.service.ts
export class MoviesSearchService {
...

  async search(  
    text: string,  
    targets: string[],  
    offset?: number,  
    limit?: number,  
    startId = 0,  
  ) {  
    let separateCount = 0;  
    if (startId) {  
      separateCount = await this.count(text, targets);  
    }  
    const params: RequestParams.Search = {  
      index: this.index,  
      from: offset,  
      size: limit,  
      body: {  
        query: {  
          multi_match: {  
            query: text,  
            fields: targets,  
          },  
        },  
      },  
    };  
    const { body: result } = await this.elasticsearchService.search(params);  
    const count = result.hits.total.value;  
    const results = result.hits.hits.map((item) => item._source);  
    return {  
      count: startId ? separateCount : count,  
      results,  
    };  
  }
...
}

// movie-seed.service.ts
import { Mistral } from "@mistralai/mistralai";

// movie-seed.service.ts
async run() {  
  // If index doesn't exist, create it and seed the database  
  const exists = await this.moviesSearchService.existsIndex();  
  if (!exists.body) {  
    await this.moviesSearchService.createIndex();  
    await this.moviesSearchService.statusIndex();  
    // Fetch movie titles from our relational database  
    const insertedMovies = await this.repository.find({});  
    // Add MistralAI Embeddings  
    //    
    // Note: In a production app, this should probably be done in the background,      // along with an update of the document in Elasticsearch/OpenSearch.    
    const aiClient = new Mistral({  
      apiKey: process.env.MISTRALAI_API_KEY,  
    }); 
    const embeddingRequests: Array> = [];  
    const moviesLen = insertedMovies.length;  
    const rateLimiter = new RateLimiterMemory({  
      points: 6, // 6 points  
      duration: 1, // Per second  
    });  

    for (const movie of insertedMovies) {  
      const index: number = insertedMovies.indexOf(movie);  
      let shouldRetry = true;  

      while (shouldRetry) {  
        await rateLimiter  
          .consume('aiEmbedding', 3)  
          .then(() => {  
            shouldRetry = false;  
            Logger.log(  
              'AI Embedding: processing ' +  
                movie.title +  
                ' (' +  
                (index + 1).toString() +  
                ' of ' +  
                moviesLen +  
                ')',  
            );  

            embeddingRequests.push(  
              this.generateScriptEmbeddings(aiClient, movie),  
            );  
          })  
          .catch(async (rlres: RateLimiterRes) => {  
            // Not enough points, wait for the specified duration  
            await new Promise((resolve) =>  
              setTimeout(resolve, rlres.msBeforeNext),  
            );  
          });  
      }  
    }  

  Promise.all(embeddingRequests)  
    .then(async () => {  
      // Index the movies discovered  
      await this.moviesSearchService.indexMovies(insertedMovies);  
      Logger.log(  
        'Completed indexing of movies (' +  
          insertedMovies.length.toString() +  
          ' total)',  
      );  
    })  
    .catch((err) => {  
      Logger.log('Encountered fatal error during AI embedding: ', err);  
    });  
  Logger.log('Search seeding complete!');
  }  
}  

// movie-seed.service.ts
export class MovieSeedService {
  ...
  async generateScriptEmbeddings(  
    aiClient: Mistral,  
    movie: MovieEntity,  
    numCharsAllowed: number = 15000,  
  ) {  
    const script_embedding = await aiClient.embeddings.create({  
      model: 'mistral-embed',  
      inputs: movie.script.slice(0, numCharsAllowed),  
    });

    // We're only requesting one embedding; so there should only be one entry!  
    if (script_embedding !== undefined && script_embedding.data.length === 1) { 
      if (Array.isArray(script_embedding.data[0].embedding)) { 
        movie.script_embedding_vector = script_embedding.data[0].embedding;
      }  
    }  
  }
}

// query-movie.dto.ts
 export enum SearchTarget {
   Title = 'title',
   Script = 'script',
+  ScriptEmbeddingVector = 'script_embedding_vector',
 }

import { Mistral } from '@mistralai/mistralai';
import { SearchTarget } from './dto/query-movie.dto';

...

@Injectable()  
export class MoviesSearchService {  
  ...

  async search(  
    text: string,  
    targets: string[],  
    offset?: number,  
    limit?: number,  
    startId = 0,  
  ) {  
    let separateCount = 0;  
    if (startId) {  
      separateCount = await this.count(text, targets);  
    }  

    // Default query handling is to perform a text query  
    let query: Record = {  
      multi_match: {  
        query: text,  
        fields: targets,  
      },  
    };  

    // If this is a query targeting the script embedding vector, adjust  
    // the query accordingly!    
    if (  
      targets.length === 1 &&  
      targets.includes(SearchTarget.ScriptEmbeddingVector)  
    ) {  
      const aiClient = new Mistral({  
        apiKey: process.env.MISTRALAI_API_KEY,  
      });  

      query = {  
        knn: {},  
      };  

      query['knn'][SearchTarget.ScriptEmbeddingVector] = {  
        vector: await this.generateQueryEmbeddings(aiClient, text),  
        k: 5,  
      };  
    }  
    const params: RequestParams.Search = {  
      index: this.index,  
      from: offset,  
      size: limit,  
      _source_excludes: [  
        SearchTarget.Script,  
        SearchTarget.ScriptEmbeddingVector,  
      ],  
      body: {  
        query: query,  
      },  
    };  
    const { body: result } = await this.elasticsearchService.search(params);  
    const count = result.hits.total.value;  
    const results = result.hits.hits.map((item) => item._source);  
    return {  
      count: startId ? separateCount : count,  
      results,  
    };  
  }  
}

...

@Injectable()  
export class MoviesSearchService {  
  ...

  async generateQueryEmbeddings(  
    aiClient: Mistral,  
    query: string,  
    numCharsAllowed: number = 15000,  
  ): Promise {  
    const query_embedding = await aiClient.embeddings.create({  
      model: 'mistral-embed',  
      inputs: query.slice(0, numCharsAllowed),  
    });  

    // We're only requesting one embedding; so there should only be one entry!  
    if (query_embedding !== undefined && query_embedding.data.length === 1) {  
      if (Array.isArray(query_embedding.data[0].embedding)) {  
        return Promise.resolve(query_embedding.data[0].embedding);  
      }  
    }  
    return Promise.resolve(null);  
  }  
}

// movies.service.ts

export class MoviesService {
  async search(...) {
    ...
    const data = await this.movieRepository.find({
+     select: ['id', 'title'],
      where: { id: In(ids) },
    });
  }
}

curl "localhost:3000/api/v1/movies?search=element&targets[]=script_embedding_vector"

{
  "data": [
    {
      "id": 1240,
      "title": "the fifth element",
    },
    {
      "id": 1329,
      "title": "invaders from mars",
    },
    {
      "id": 1444,
      "title": "the time machine",
    },
    {
      "id": 1513,
      "title": "brazil",
    },
    {
      "id": 1684,
      "title": "the nightmare before christmas",
    }
  ],
  "count": 5
}