跨多个索引的多个字段上的 Elasticsearch 聚合
Posted
技术标签:
【中文标题】跨多个索引的多个字段上的 Elasticsearch 聚合【英文标题】：Elasticsearch aggregation on multiple fields across multiple indexes 【发布时间】：2015-12-20 05:09:18 【问题描述】：我有两个索引 - 一个用于 Application 模型，另一个用于 Databases 模型（多对多关系）。
每个文档都被非规范化以包含来自其他模型的属性
Application
|_ vendor_name
|_ databases
|_ db_1
|_ db_2
Database
|_ database_applications
|_ app_1
|_vendor_name
|_ app_2
|_ vendor_name
对供应商名称执行多索引搜索 - 我似乎从两个索引中都得到了正确的结果。
挑战是在 vendor_name 字段上正确聚合
当结果仅来自数据库时，使用以下聚合似乎有效。我也试过field: '*vendor_name'，但似乎没有用。
我错过了什么？模型应该改变吗？
aggregation:
vendor_name:
terms:
field: "database_applications.vendor_name"
更新 1：
根据@Andrie-Stefan - 这是两个索引映射的更准确表示（缩写为缩写）：
数据库
"company-company_databases":
"aliases": ,
"mappings":
"company_database":
"properties":
"company_applications":
"properties":
"application_id":
"type": "long"
,
"application_name":
"type": "string"
,
"business_owner":
"type": "string"
,
"company_system_applications":
"properties":
"allow_add_request":
"type": "string"
,
"allow_remove_request":
"type": "string"
,
"asset_type":
"type": "string"
,
"company_application_id":
"type": "long"
,
"company_application_name":
"type": "string"
,
"company_business_owner":
"type": "string"
,
"company_division_id":
"type": "long"
,
"company_it_app_steward":
"type": "string"
,
"company_notes":
"type": "string"
,
"company_system_id":
"type": "long"
,
"company_vendor":
"type": "string"
,
"id":
"type": "long"
,
"it_app_steward":
"type": "string"
,
"it_owner":
"type": "string"
,
"last_modified":
"type": "string"
,
"last_modified_by":
"type": "string"
,
"media_location":
"type": "string"
,
"media_source":
"type": "string"
,
"name":
"type": "string"
,
"owned_by":
"type": "string"
,
"status":
"type": "string"
,
"status_id":
"type": "long"
,
"system_application":
"properties":
"division":
"type": "string"
,
"id":
"type": "long"
,
"name":
"type": "string"
,
"owner_id":
"type": "string"
,
"status":
"type": "string"
,
"steward_id":
"type": "string"
,
"vendor_name":
"type": "string"
,
"vendor_url_web_site":
"type": "string"
,
"version":
"type": "string"
,
"vendor_name":
"type": "string",
"fields":
"raw":
"type": "string",
"index": "not_analyzed"
,
"version":
"type": "string"
,
"division_id":
"type": "long"
,
"it_app_steward":
"type": "string"
,
"notes":
"type": "string"
,
"software_inventory_id":
"type": "long"
,
"vendor":
"type": "string"
,
"company_instances":
"properties":
"business_environment_id":
"type": "long"
,
"cgi_service_id":
"type": "long"
,
"char_set":
"type": "string"
,
"confirmed_license_purchase_dt":
"type": "string"
,
"company_server":
"properties":
"business_environment_id":
"type": "long"
,
"division_id":
"type": "long"
,
"domain":
"type": "string"
,
"hw_platform_id":
"type": "long"
,
"ip_address":
"type": "string"
,
"location_id":
"type": "long"
,
"no_of_cpu":
"type": "long"
,
"notes":
"type": "string"
,
"os_platform_id":
"type": "long"
,
"os_version":
"type": "string"
,
"server_id":
"type": "long"
,
"server_name":
"type": "string"
,
"description":
"type": "string"
,
"division_id":
"type": "long"
,
"edition_id":
"type": "long"
,
"instance_id":
"type": "long"
,
"instance_name":
"type": "string"
,
"itap_have_access":
"type": "string"
,
"listener_port":
"type": "long"
,
"notes":
"type": "string"
,
"patch_number":
"type": "string"
,
"rdbms_type_id":
"type": "long"
,
"server_id":
"type": "long"
,
"service_level_id":
"type": "long"
,
"version":
"type": "string"
,
"db_security_model_id":
"type": "long"
,
"schema_or_db":
"type": "string"
,
"schema_or_db_id":
"type": "long"
,
"schema_or_db_type_id":
"type": "long"
,
"settings":
"index":
"creation_date": "1442976578465",
"uuid": "TxQZoNSpR5qa2Y2ERZzuYw",
"number_of_replicas": "1",
"number_of_shards": "5",
"version":
"created": "1070299"
,
"warmers":
应用
"applications":
"aliases": ,
"mappings":
"application":
"properties":
"application_view":
"properties":
"app_name":
"type": "string"
,
"app_status":
"type": "string"
,
"app_steward_name":
"type": "string"
,
"app_suite":
"type": "string"
,
"app_vendor_name":
"type": "string"
,
"app_version":
"type": "string"
,
"assignment_group":
"type": "string"
,
"business_domain_name":
"type": "string"
,
"exception":
"type": "string"
,
"id":
"type": "long"
,
"it_owner_name":
"type": "string"
,
"service_level":
"type": "string"
,
"assignment_group":
"type": "string"
,
"company_databases":
"properties":
"backup_history_info":
"type": "string"
,
"company_applications":
"properties":
"alternate_name":
"type": "string"
,
"application_id":
"type": "long"
,
"application_name":
"type": "string"
,
"business_owner":
"type": "string"
,
"company_system_applications":
"properties":
"aka":
"type": "string"
,
"allow_add_request":
"type": "string"
,
"allow_remove_request":
"type": "string"
,
"asset_type":
"type": "string"
,
"contract_number":
"type": "string"
,
"cost_level":
"type": "string"
,
"company_alternate_name":
"type": "string"
,
"company_application_id":
"type": "long"
,
"company_application_name":
"type": "string"
,
"company_business_owner":
"type": "string"
,
"company_division_id":
"type": "long"
,
"company_it_app_steward":
"type": "string"
,
"company_notes":
"type": "string"
,
"company_system_id":
"type": "long"
,
"company_vendor":
"type": "string"
,
"description":
"type": "string"
,
"display_in_catalog":
"type": "string"
,
"id":
"type": "long"
,
"inform_of_removal":
"type": "string"
,
"is_restricted":
"type": "string"
,
"it_app_steward":
"type": "string"
,
"it_owner":
"type": "string"
,
"last_modified":
"type": "string"
,
"last_modified_by":
"type": "string"
,
"media_location":
"type": "string"
,
"media_source":
"type": "string"
,
"name":
"type": "string"
,
"os_environment":
"type": "string"
,
"owned_by":
"type": "string"
,
"retirement_date":
"type": "date",
"format": "dateOptionalTime"
,
"status":
"type": "string"
,
"status_id":
"type": "long"
,
"suite_name":
"type": "string"
,
"system_application":
"properties":
"assignment_group":
"type": "string"
,
"division":
"type": "string"
,
"id":
"type": "long"
,
"name":
"type": "string"
,
"owner_id":
"type": "string"
,
"status":
"type": "string"
,
"steward_id":
"type": "string"
,
"suite":
"type": "string"
,
"vendor_name":
"type": "string"
,
"vendor_url_web_site":
"type": "string"
,
"version":
"type": "string"
,
"vendor_name":
"type": "string"
,
"version":
"type": "string"
,
"division_id":
"type": "long"
,
"it_app_steward":
"type": "string"
,
"notes":
"type": "string"
,
"software_inventory_id":
"type": "long"
,
"vendor":
"type": "string"
,
"company_instances":
"properties":
"business_environment_id":
"type": "long"
,
"cgi_service_id":
"type": "long"
,
"char_set":
"type": "string"
,
"confirmed_license_purchase_dt":
"type": "string"
,
"company_server":
"properties":
"business_environment_id":
"type": "long"
,
"division_id":
"type": "long"
,
"domain":
"type": "string"
,
"hw_platform_id":
"type": "long"
,
"ip_address":
"type": "string"
,
"location_id":
"type": "long"
,
"no_of_cpu":
"type": "long"
,
"notes":
"type": "string"
,
"os_platform_id":
"type": "long"
,
"os_version":
"type": "string"
,
"server_id":
"type": "long"
,
"server_name":
"type": "string"
,
"description":
"type": "string"
,
"division_id":
"type": "long"
,
"edition_id":
"type": "long"
,
"instance_id":
"type": "long"
,
"instance_name":
"type": "string"
,
"itap_have_access":
"type": "string"
,
"listener_port":
"type": "long"
,
"location_id":
"type": "long"
,
"notes":
"type": "string"
,
"patch_number":
"type": "string"
,
"rdbms_type_id":
"type": "long"
,
"server_id":
"type": "long"
,
"service_level_id":
"type": "long"
,
"version":
"type": "string"
,
"db_security_model_id":
"type": "long"
,
"notes":
"type": "string"
,
"schema_or_db":
"type": "string"
,
"schema_or_db_id":
"type": "long"
,
"schema_or_db_type_id":
"type": "long"
,
"division":
"type": "string",
"fields":
"raw":
"type": "string",
"index": "not_analyzed"
,
"id":
"type": "long"
,
"name":
"type": "string"
,
"owner":
"properties":
"email_address":
"type": "string"
,
"id":
"type": "long"
,
"name":
"type": "string",
"fields":
"raw":
"type": "string",
"index": "not_analyzed"
,
"search_type":
"type": "string"
,
"user_id":
"type": "string"
,
"owner_id":
"type": "string"
,
"status":
"type": "string",
"fields":
"raw":
"type": "string",
"index": "not_analyzed"
,
"steward":
"properties":
"email_address":
"type": "string"
,
"id":
"type": "long"
,
"name":
"type": "string",
"fields":
"raw":
"type": "string",
"index": "not_analyzed"
,
"search_type":
"type": "string"
,
"user_id":
"type": "string"
,
"steward_id":
"type": "string"
,
"suite":
"type": "string"
,
"vendor_name":
"type": "string",
"fields":
"raw":
"type": "string",
"index": "not_analyzed"
,
"vendor_url_web_site":
"type": "string"
,
"version":
"type": "string"
,
"settings":
"index":
"creation_date": "1442970067540",
"uuid": "O7DTaCSESbqhjJpv62T0Wg",
"number_of_replicas": "1",
"number_of_shards": "5",
"version":
"created": "1070299"
,
"warmers":
【问题讨论】：
如果Application 是一个索引那么vendor_name 是一个类型还是你忘记在图片中包含类型名称？
你能分享两个索引的映射吗？
【参考方案1】：
只有在名称相同的情况下，才能跨索引聚合字段。聚合字段没有通配符语法。
这是您的映射当前定义的内容：
索引：company-company_databases
类型：company_database
字段名称：
company_applications.company_system_applications.vendor_name
company_applications.company_system_applications.system_application.vendor_name
索引：applications
类型：application
字段名称：
company_databases.company_applications.company_system_applications.vendor_name
company_databases.company_applications.company_system_applications.system_application.vendor_name
就 Elasticsearch 而言，这些字段没有任何共同点（即使路径的一部分是 vendor_name）。
如果您的目标是在跨越两个索引的查询中聚合 vendor_name，请考虑重组您的索引/映射以实现此目的。
请注意，Elasticsearch doesn't model many-to-many relationships
如果您可以避免跨应用程序复制数据库信息，您也许可以将您的关系重新表述为层次结构，例如：
索引：applications
--
类型：application
字段：vendor_name等...
--
类型：database_application
字段：vendor_name、databases.<inner fields> 等...
--
然后，您可以在同一字段路径 vendor_name 上跨类型进行聚合，另外还可以查询单个 applications 索引。
【讨论】：
以上是关于跨多个索引的多个字段上的 Elasticsearch 聚合的主要内容，如果未能解决你的问题，请参考以下文章
多台NGINX服务器 ELK部署Filebeat的fields字段+Logstash的if 字段 采集多个日志文件写入elasticsearch 7.6.2 变更为不同索引名 自定义索引 index