Implementing Fast Search with Elasticsearch and NestJS

Peruse My Scroll of Curious Writs

Learn how to implement fast and scalable search functionality using Elasticsearch with NestJS, including best practices for indexing, searching, and maintaining data synchronization.

Elasticsearch | NestJS | TypeScript | Search | Performance | Database

Elasticsearch is a powerful search engine that can significantly improve search performance in your applications. In this guide, we'll explore how to integrate Elasticsearch with NestJS to create fast, scalable, and feature-rich search functionality.

Setting Up Elasticsearch with NestJS

First, let's set up our NestJS project with Elasticsearch. We'll use the official Elasticsearch client and create a dedicated module for search functionality.

npm install @nestjs/elasticsearch @elastic/elasticsearch        

Creating the Elasticsearch Module

// elasticsearch.module.ts
import { Module } from '@nestjs/common';
import { ElasticsearchModule } from '@nestjs/elasticsearch';
import { ConfigModule, ConfigService } from '@nestjs/config';

@Module({
  imports: [
    ElasticsearchModule.registerAsync({
      imports: [ConfigModule],
      useFactory: async (configService: ConfigService) => ({
        node: configService.get('ELASTICSEARCH_NODE'),
        auth: {
          username: configService.get('ELASTICSEARCH_USERNAME'),
          password: configService.get('ELASTICSEARCH_PASSWORD'),
        },
        tls: {
          rejectUnauthorized: false
        }
      }),
      inject: [ConfigService],
    }),
  ],
  exports: [ElasticsearchModule],
})
export class SearchModule {}        

Defining Search Interfaces

// search.interface.ts
export interface SearchableProduct {
  id: number;
  name: string;
  description: string;
  price: number;
  category: string;
  tags: string[];
  createdAt: Date;
}

export interface SearchResponse<T> {
  hits: {
    total: {
      value: number;
    };
    hits: Array<{
      _source: T;
      _score: number;
    }>;
  };
}        

Implementing the Search Service

// search.service.ts
import { Injectable } from '@nestjs/common';
import { ElasticsearchService } from '@nestjs/elasticsearch';
import { SearchableProduct, SearchResponse } from './search.interface';

@Injectable()
export class SearchService {
  private readonly index = 'products';

  constructor(private readonly elasticsearchService: ElasticsearchService) {}

  async indexProduct(product: SearchableProduct) {
    return this.elasticsearchService.index<SearchableProduct>({
      index: this.index,
      document: {
        id: product.id,
        name: product.name,
        description: product.description,
        price: product.price,
        category: product.category,
        tags: product.tags,
        createdAt: product.createdAt
      },
    });
  }

  async search(text: string) {
    const { hits } = await this.elasticsearchService.search<SearchResponse<SearchableProduct>>({
      index: this.index,
      query: {
        multi_match: {
          query: text,
          fields: ['name^3', 'description^2', 'category', 'tags'],
          fuzziness: 'AUTO'
        }
      },
      sort: [
        { _score: { order: 'desc' } },
        { createdAt: { order: 'desc' } }
      ],
      highlight: {
        fields: {
          name: {},
          description: {}
        }
      }
    });

    return hits.hits.map(hit => ({
      ...hit._source,
      score: hit._score,
      highlights: hit.highlight
    }));
  }

  async searchWithFilters(params: {
    text?: string;
    category?: string;
    minPrice?: number;
    maxPrice?: number;
    tags?: string[];
    page?: number;
    limit?: number;
  }) {
    const { text, category, minPrice, maxPrice, tags, page = 1, limit = 10 } = params;

    const must: any[] = [];
    const filter: any[] = [];

    if (text) {
      must.push({
        multi_match: {
          query: text,
          fields: ['name^3', 'description^2', 'category', 'tags'],
          fuzziness: 'AUTO'
        }
      });
    }

    if (category) {
      filter.push({ term: { category } });
    }

    if (minPrice !== undefined || maxPrice !== undefined) {
      filter.push({
        range: {
          price: {
            ...(minPrice !== undefined && { gte: minPrice }),
            ...(maxPrice !== undefined && { lte: maxPrice })
          }
        }
      });
    }

    if (tags?.length) {
      filter.push({ terms: { tags } });
    }

    const { hits } = await this.elasticsearchService.search<SearchResponse<SearchableProduct>>({
      index: this.index,
      query: {
        bool: {
          must,
          filter
        }
      },
      sort: [
        { _score: { order: 'desc' } },
        { createdAt: { order: 'desc' } }
      ],
      from: (page - 1) * limit,
      size: limit,
      highlight: {
        fields: {
          name: {},
          description: {}
        }
      }
    });

    return {
      items: hits.hits.map(hit => ({
        ...hit._source,
        score: hit._score,
        highlights: hit.highlight
      })),
      total: hits.total.value,
      page,
      limit,
      pages: Math.ceil(hits.total.value / limit)
    };
  }
}        

Implementing Data Synchronization

To keep Elasticsearch in sync with your primary database, you can use event listeners or observers. Here's an example using TypeORM subscribers:

// product.subscriber.ts
import { Connection, EntitySubscriberInterface, EventSubscriber, InsertEvent, UpdateEvent, RemoveEvent } from 'typeorm';
import { Injectable } from '@nestjs/common';
import { SearchService } from './search.service';
import { Product } from './product.entity';

@Injectable()
@EventSubscriber()
export class ProductSubscriber implements EntitySubscriberInterface<Product> {
  constructor(
    connection: Connection,
    private readonly searchService: SearchService
  ) {
    connection.subscribers.push(this);
  }

  listenTo() {
    return Product;
  }

  async afterInsert(event: InsertEvent<Product>) {
    await this.searchService.indexProduct(this.toSearchableProduct(event.entity));
  }

  async afterUpdate(event: UpdateEvent<Product>) {
    if (event.entity) {
      await this.searchService.indexProduct(this.toSearchableProduct(event.entity));
    }
  }

  async beforeRemove(event: RemoveEvent<Product>) {
    if (event.entityId) {
      await this.searchService.remove(event.entityId);
    }
  }

  private toSearchableProduct(product: Product): SearchableProduct {
    return {
      id: product.id,
      name: product.name,
      description: product.description,
      price: product.price,
      category: product.category,
      tags: product.tags,
      createdAt: product.createdAt
    };
  }
}        

Using the Search Service in Controllers

// search.controller.ts
import { Controller, Get, Query } from '@nestjs/common';
import { SearchService } from './search.service';

@Controller('search')
export class SearchController {
  constructor(private readonly searchService: SearchService) {}

  @Get()
  async search(
    @Query('q') query: string,
    @Query('category') category?: string,
    @Query('minPrice') minPrice?: number,
    @Query('maxPrice') maxPrice?: number,
    @Query('tags') tags?: string,
    @Query('page') page?: number,
    @Query('limit') limit?: number,
  ) {
    return this.searchService.searchWithFilters({
      text: query,
      category,
      minPrice,
      maxPrice,
      tags: tags?.split(','),
      page,
      limit
    });
  }
}        

Best Practices and Optimization

• Use bulk operations for indexing multiple documents
• Implement proper index mapping for better search results
• Use index aliases for zero-downtime reindexing
• Implement caching for frequently searched queries
• Use scroll API for large result sets
• Monitor search performance and optimize queries

Implementing Bulk Operations

// search.service.ts
async bulkIndex(products: SearchableProduct[]) {
  const operations = products.flatMap(product => [
    { index: { _index: this.index } },
    {
      id: product.id,
      name: product.name,
      description: product.description,
      price: product.price,
      category: product.category,
      tags: product.tags,
      createdAt: product.createdAt
    }
  ]);

  return this.elasticsearchService.bulk({ operations });
}        

Index Mapping Configuration

// search.service.ts
async createIndex() {
  const index = this.index;
  const exists = await this.elasticsearchService.indices.exists({ index });

  if (!exists) {
    await this.elasticsearchService.indices.create({
      index,
      mappings: {
        properties: {
          id: { type: 'integer' },
          name: { 
            type: 'text',
            analyzer: 'standard',
            fields: {
              keyword: {
                type: 'keyword',
                ignore_above: 256
              }
            }
          },
          description: { type: 'text' },
          price: { type: 'float' },
          category: { type: 'keyword' },
          tags: { type: 'keyword' },
          createdAt: { type: 'date' }
        }
      },
      settings: {
        analysis: {
          analyzer: {
            standard: {
              type: 'standard',
              stopwords: '_english_'
            }
          }
        }
      }
    });
  }
}        

By following these patterns and implementing proper optimization strategies, you can create a powerful search functionality that scales well with your application's growth. Remember to monitor your Elasticsearch cluster's performance and adjust settings based on your specific use case.

- AB The Tome of Ingenious Craft