ElasticSearch全文搜索实战

前言

ElasticSearch是目前最流行的开源搜索引擎，广泛应用于日志分析、全文搜索、数据分析等场景。本文将系统讲解ES的核心概念、使用技巧和最佳实践，帮助你构建高性能的搜索系统。

一、ElasticSearch基础

1.1 核心概念

1
2
3
4
5
6
7
索引(Index)     ≈ 数据库
类型(Type)      ≈ 表(7.x后废弃)
文档(Document)  ≈ 行
字段(Field)     ≈ 列
映射(Mapping)   ≈ 表结构
分片(Shard)     = 数据分布单元
副本(Replica)   = 数据备份

1.2 安装启动

1
2
3
4
5
6
7
8
9
10
11
# Docker方式
docker run -d \
  --name elasticsearch \
  -p 9200:9200 \
  -p 9300:9300 \
  -e "discovery.type=single-node" \
  -e "ES_JAVA_OPTS=-Xms512m -Xmx512m" \
  elasticsearch:8.8.0

# 验证
curl http://localhost:9200

1.3 基本操作

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
# 创建索引
PUT /products
{
  "settings": {
    "number_of_shards": 3,
    "number_of_replicas": 1
  }
}

# 添加文档
POST /products/_doc/1
{
  "name": "iPhone 15 Pro",
  "price": 999,
  "category": "Electronics",
  "brand": "Apple",
  "tags": ["smartphone", "5G"],
  "created_at": "2024-04-08"
}

# 获取文档
GET /products/_doc/1

# 更新文档
POST /products/_update/1
{
  "doc": {
    "price": 899
  }
}

# 删除文档
DELETE /products/_doc/1

# 删除索引
DELETE /products

二、映射(Mapping)设计

2.1 字段类型

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
PUT /products
{
  "mappings": {
    "properties": {
      "name": {
        "type": "text",           # 全文搜索
        "analyzer": "ik_max_word", # 分词器
        "fields": {
          "keyword": {             # 多字段mapping
            "type": "keyword"      # 精确匹配
          }
        }
      },
      "price": {
        "type": "double"
      },
      "stock": {
        "type": "integer"
      },
      "category": {
        "type": "keyword"         # 不分词
      },
      "description": {
        "type": "text",
        "analyzer": "ik_smart"
      },
      "tags": {
        "type": "keyword"         # 数组
      },
      "location": {
        "type": "geo_point"       # 地理位置
      },
      "created_at": {
        "type": "date",
        "format": "yyyy-MM-dd HH:mm:ss||yyyy-MM-dd||epoch_millis"
      },
      "is_active": {
        "type": "boolean"
      }
    }
  }
}

2.2 动态映射

1
2
3
4
5
6
7
8
9
10
11
12
PUT /products/_mapping
{
  "dynamic": "strict",  # true | false | strict
  "properties": {
    ...
  }
}

# dynamic模式：
# - true: 自动添加新字段
# - false: 忽略新字段
# - strict: 拒绝新字段，抛出异常

三、查询DSL

3.1 基础查询

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
# Match Query - 全文搜索
GET /products/_search
{
  "query": {
    "match": {
      "name": "iPhone Pro"
    }
  }
}

# Multi-Match - 多字段搜索
GET /products/_search
{
  "query": {
    "multi_match": {
      "query": "smartphone",
      "fields": ["name", "description"]
    }
  }
}

# Term Query - 精确匹配
GET /products/_search
{
  "query": {
    "term": {
      "category.keyword": "Electronics"
    }
  }
}

# Range Query - 范围查询
GET /products/_search
{
  "query": {
    "range": {
      "price": {
        "gte": 500,
        "lte": 1000
      }
    }
  }
}

3.2 复合查询

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# Bool Query
GET /products/_search
{
  "query": {
    "bool": {
      "must": [                    # 必须匹配(AND)
        { "match": { "name": "iPhone" } }
      ],
      "filter": [                  # 过滤(不计算分数)
        { "term": { "category": "Electronics" } },
        { "range": { "price": { "lte": 1000 } } }
      ],
      "should": [                  # 应该匹配(OR)
        { "term": { "brand": "Apple" } }
      ],
      "must_not": [                # 必须不匹配(NOT)
        { "term": { "is_active": false } }
      ],
      "minimum_should_match": 1
    }
  }
}

3.3 分页和排序

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
GET /products/_search
{
  "query": {
    "match_all": {}
  },
  "from": 0,        # 起始位置
  "size": 20,       # 返回数量
  "sort": [
    { "price": "asc" },
    { "created_at": "desc" },
    "_score"        # 相关性分数
  ],
  "_source": ["name", "price"]  # 只返回特定字段
}

# 深度分页优化 - Search After
GET /products/_search
{
  "size": 10,
  "sort": [
    { "created_at": "desc" },
    { "_id": "desc" }
  ],
  "search_after": ["2024-04-08", "100"]
}

3.4 高亮显示

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
GET /products/_search
{
  "query": {
    "match": {
      "name": "iPhone"
    }
  },
  "highlight": {
    "fields": {
      "name": {
        "pre_tags": ["<em>"],
        "post_tags": ["</em>"]
      }
    }
  }
}

四、聚合分析

4.1 指标聚合

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
# 统计聚合
GET /products/_search
{
  "size": 0,
  "aggs": {
    "avg_price": {
      "avg": { "field": "price" }
    },
    "max_price": {
      "max": { "field": "price" }
    },
    "min_price": {
      "min": { "field": "price" }
    },
    "sum_stock": {
      "sum": { "field": "stock" }
    },
    "stats_price": {
      "stats": { "field": "price" }
    }
  }
}

4.2 桶聚合

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
# 按分类分组
GET /products/_search
{
  "size": 0,
  "aggs": {
    "group_by_category": {
      "terms": {
        "field": "category.keyword",
        "size": 10,
        "order": { "_count": "desc" }
      },
      "aggs": {
        "avg_price": {
          "avg": { "field": "price" }
        }
      }
    }
  }
}

# 日期直方图
GET /products/_search
{
  "size": 0,
  "aggs": {
    "sales_over_time": {
      "date_histogram": {
        "field": "created_at",
        "calendar_interval": "month"
      },
      "aggs": {
        "total_sales": {
          "sum": { "field": "price" }
        }
      }
    }
  }
}

# 价格区间
GET /products/_search
{
  "size": 0,
  "aggs": {
    "price_ranges": {
      "range": {
        "field": "price",
        "ranges": [
          { "to": 500 },
          { "from": 500, "to": 1000 },
          { "from": 1000 }
        ]
      }
    }
  }
}

五、中文分词

5.1 安装IK分词器

1
2
3
4
5
# 在线安装
elasticsearch-plugin install https://github.com/medcl/elasticsearch-analysis-ik/releases/download/v8.8.0/elasticsearch-analysis-ik-8.8.0.zip

# 重启ES
docker restart elasticsearch

5.2 分词器测试

1
2
3
4
5
6
7
8
9
10
11
12
13
# ik_max_word - 最细粒度拆分
POST /_analyze
{
  "analyzer": "ik_max_word",
  "text": "中华人民共和国国歌"
}

# ik_smart - 最粗粒度拆分
POST /_analyze
{
  "analyzer": "ik_smart",
  "text": "中华人民共和国国歌"
}

5.3 自定义词典

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# config/analysis-ik/IKAnalyzer.cfg.xml
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE properties SYSTEM "http://java.sun.com/dtd/properties.dtd">
<properties>
    <comment>IK Analyzer 扩展配置</comment>
    <entry key="ext_dict">custom.dic</entry>
    <entry key="ext_stopwords">stopword.dic</entry>
</properties>

# config/analysis-ik/custom.dic
iPhone
安卓
小米

# 重启生效

六、性能优化

6.1 索引优化

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
PUT /products/_settings
{
  "index": {
    # 刷新间隔
    "refresh_interval": "30s",  # 默认1s
    
    # 副本数量
    "number_of_replicas": 1,
    
    # 最大结果窗口
    "max_result_window": 10000,
    
    # 分片分配
    "routing": {
      "allocation": {
        "total_shards_per_node": 2
      }
    }
  }
}

# 批量导入时临时禁用刷新
PUT /products/_settings
{
  "refresh_interval": "-1",
  "number_of_replicas": 0
}

# 导入完成后恢复
PUT /products/_settings
{
  "refresh_interval": "1s",
  "number_of_replicas": 1
}
POST /products/_forcemerge

6.2 查询优化

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
# 使用filter代替query(不计算分数)
GET /products/_search
{
  "query": {
    "bool": {
      "filter": [
        { "term": { "category": "Electronics" } }
      ]
    }
  }
}

# 禁用分数计算
GET /products/_search
{
  "query": {
    "bool": {
      "filter": [
        { "match": { "name": "iPhone" } }
      ]
    }
  },
  "track_scores": false
}

# 控制返回字段
GET /products/_search
{
  "_source": {
    "includes": ["name", "price"],
    "excludes": ["description"]
  }
}

6.3 批量操作

1
2
3
4
5
6
7
8
9
# Bulk API
POST /_bulk
{ "index": { "_index": "products", "_id": "1" } }
{ "name": "Product 1", "price": 100 }
{ "index": { "_index": "products", "_id": "2" } }
{ "name": "Product 2", "price": 200 }
{ "update": { "_index": "products", "_id": "1" } }
{ "doc": { "price": 150 } }
{ "delete": { "_index": "products", "_id": "3" } }

七、Java客户端

7.1 Spring Data ElasticSearch

1
2
3
4
5
<!-- pom.xml -->
<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
// 实体类
@Document(indexName = "products")
public class Product {
    @Id
    private String id;
    
    @Field(type = FieldType.Text, analyzer = "ik_max_word")
    private String name;
    
    @Field(type = FieldType.Double)
    private Double price;
    
    @Field(type = FieldType.Keyword)
    private String category;
    
    @Field(type = FieldType.Date, format = DateFormat.date_time)
    private Date createdAt;
    
    // getter/setter
}

// Repository
public interface ProductRepository extends ElasticsearchRepository<Product, String> {
    List<Product> findByName(String name);
    
    List<Product> findByPriceBetween(Double min, Double max);
    
    List<Product> findByCategoryAndPriceGreaterThan(String category, Double price);
}

// Service
@Service
public class ProductService {
    @Autowired
    private ProductRepository productRepository;
    
    @Autowired
    private ElasticsearchRestTemplate elasticsearchTemplate;
    
    // 保存
    public Product save(Product product) {
        return productRepository.save(product);
    }
    
    // 搜索
    public List<Product> search(String keyword) {
        Criteria criteria = new Criteria("name").matches(keyword)
            .or("description").matches(keyword);
        
        Query query = new CriteriaQuery(criteria);
        
        SearchHits<Product> hits = elasticsearchTemplate.search(query, Product.class);
        
        return hits.stream()
            .map(SearchHit::getContent)
            .collect(Collectors.toList());
    }
    
    // 复杂查询
    public Page<Product> complexSearch(String keyword, String category, Double minPrice) {
        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
        
        if (keyword != null) {
            boolQuery.must(QueryBuilders.multiMatchQuery(keyword, "name", "description"));
        }
        
        if (category != null) {
            boolQuery.filter(QueryBuilders.termQuery("category", category));
        }
        
        if (minPrice != null) {
            boolQuery.filter(QueryBuilders.rangeQuery("price").gte(minPrice));
        }
        
        NativeSearchQuery query = new NativeSearchQueryBuilder()
            .withQuery(boolQuery)
            .withPageable(PageRequest.of(0, 10))
            .withSort(SortBuilders.fieldSort("price").order(SortOrder.ASC))
            .build();
        
        SearchHits<Product> hits = elasticsearchTemplate.search(query, Product.class);
        
        return SearchHitSupport.searchPageFor(hits, query.getPageable());
    }
}

八、实战案例

8.1 电商搜索

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
// 商品搜索
GET /products/_search
{
  "query": {
    "bool": {
      "must": [
        {
          "multi_match": {
            "query": "iPhone 手机",
            "fields": ["name^3", "description"],  # name权重3倍
            "type": "best_fields",
            "minimum_should_match": "75%"
          }
        }
      ],
      "filter": [
        { "term": { "category": "手机" } },
        { "range": { "price": { "gte": 3000, "lte": 8000 } } },
        { "term": { "brand": "苹果" } }
      ]
    }
  },
  "aggs": {
    "price_stats": {
      "stats": { "field": "price" }
    },
    "brand_count": {
      "terms": { "field": "brand.keyword" }
    }
  },
  "sort": [
    { "_score": "desc" },
    { "sales": "desc" },
    { "created_at": "desc" }
  ],
  "highlight": {
    "fields": {
      "name": {},
      "description": {}
    }
  }
}

8.2 日志分析

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
# 创建日志索引
PUT /logs-2024.04
{
  "mappings": {
    "properties": {
      "timestamp": { "type": "date" },
      "level": { "type": "keyword" },
      "message": { "type": "text" },
      "service": { "type": "keyword" },
      "host": { "type": "keyword" },
      "trace_id": { "type": "keyword" }
    }
  }
}

# 分析错误日志
GET /logs-*/_search
{
  "query": {
    "bool": {
      "filter": [
        { "term": { "level": "ERROR" } },
        {
          "range": {
            "timestamp": {
              "gte": "now-1h"
            }
          }
        }
      ]
    }
  },
  "aggs": {
    "errors_by_service": {
      "terms": { "field": "service" }
    },
    "errors_over_time": {
      "date_histogram": {
        "field": "timestamp",
        "calendar_interval": "1m"
      }
    }
  }
}

九、集群管理

9.1 集群健康

1
2
3
4
5
6
7
8
9
10
11
# 查看集群健康
GET /_cluster/health

# 查看节点信息
GET /_cat/nodes?v

# 查看索引信息
GET /_cat/indices?v

# 查看分片分配
GET /_cat/shards?v

9.2 索引别名

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# 创建别名
POST /_aliases
{
  "actions": [
    {
      "add": {
        "index": "products_v1",
        "alias": "products"
      }
    }
  ]
}

# 切换别名(零停机)
POST /_aliases
{
  "actions": [
    { "remove": { "index": "products_v1", "alias": "products" } },
    { "add": { "index": "products_v2", "alias": "products" } }
  ]
}

十、总结

ElasticSearch核心要点：

关键概念

  1. 倒排索引：快速全文搜索的核心
  2. 分片和副本：分布式存储和高可用
  3. 分词器：中文搜索的关键
  4. 聚合分析：强大的数据分析能力

最佳实践

  • 合理设计Mapping
  • 使用filter代替query
  • 控制分片数量
  • 定期优化索引
  • 监控集群健康

掌握ElasticSearch，让你的搜索功能飞起来！

参考资料