elasticsearch基础—— 复合聚合

Posted 2021-09-13 一曲广陵散

tags:

篇首语：本文由小常识网(cha138.com)小编为大家整理，主要介绍了elasticsearch基础—— 复合聚合相关的知识，希望对你有一定的参考价值。

一、参考

elasticsearch 学习系列目录——更新ing
Composite aggregation
Composite aggregation ORDER BY

二、产生原因

当有多层聚合需要时候，使用composite聚合，可以更好的分页

三、`sources`类型

sources参数中定义了，复合聚合的组成源数据

测试数据

GET kibana_sample_data_logs/_search
{
  "size": 1
}

{
  "took" : 2,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : 1.0,
    "hits" : [
      {
        "_index" : "kibana_sample_data_logs",
        "_type" : "_doc",
        "_id" : "4O9NX3kBTG9UhPTpZasD",
        "_score" : 1.0,
        "_source" : {
          "agent" : "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)",
          "bytes" : 7525,
          "clientip" : "60.103.76.51",
          "extension" : "css",
          "geo" : {
            "srcdest" : "IN:TW",
            "src" : "IN",
            "dest" : "TW",
            "coordinates" : {
              "lat" : 35.23199833,
              "lon" : -102.3990931
            }
          },
          "host" : "cdn.elastic-elastic-elastic.org",
          "index" : "kibana_sample_data_logs",
          "ip" : "60.103.76.51",
          "machine" : {
            "ram" : 2147483648,
            "os" : "ios"
          },
          "memory" : null,
          "message" : "60.103.76.51 - - [2018-08-10T10:14:00.227Z] \\"GET /styles/ads.css HTTP/1.1\\" 200 7525 \\"-\\" \\"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1; .NET CLR 1.1.4322)\\"",
          "phpmemory" : null,
          "referer" : "http://twitter.com/success/sandra-magnus",
          "request" : "/styles/ads.css",
          "response" : 200,
          "tags" : [
            "success",
            "security"
          ],
          "timestamp" : "2021-05-21T10:14:00.227Z",
          "url" : "https://cdn.elastic-elastic-elastic.org/styles/ads.css",
          "utc_time" : "2021-05-21T10:14:00.227Z",
          "event" : {
            "dataset" : "sample_web_logs"
          }
        }
      }
    ]
  }
}

3.1 `terms`

普通的terms聚合

GET kibana_sample_data_logs/_search
{
  "size": 0,
  "aggs": {
    "aggs1": {
      "terms": {
        "field": "clientip",
        "size": 3
      }
    }
  }
}

{
  "took" : 14,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "aggs1" : {
      "doc_count_error_upper_bound" : 0,
      "sum_other_doc_count" : 13919,
      "buckets" : [
        {
          "key" : "30.156.16.164",
          "doc_count" : 100
        },
        {
          "key" : "164.85.94.243",
          "doc_count" : 29
        },
        {
          "key" : "50.184.59.162",
          "doc_count" : 26
        }
      ]
    }
  }
}

composite聚合中的terms

GET kibana_sample_data_logs/_search
{
  "size": 0,
  "aggs": {
    "aggs1": {
      "composite": {
        "size": 3, 
        "sources": [
          {
            "clientipAggs": {
              "terms": {
                "field": "clientip",
                "order": "asc"
              }
            }
          }
        ]
      }
    }
  }
}

{
  "took" : 6,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "aggs1" : {
      "after_key" : {
        "clientipAggs" : "0.209.144.101"
      },
      "buckets" : [
        {
          "key" : {
            "clientipAggs" : "0.72.176.46"
          },
          "doc_count" : 14
        },
        {
          "key" : {
            "clientipAggs" : "0.207.229.147"
          },
          "doc_count" : 11
        },
        {
          "key" : {
            "clientipAggs" : "0.209.144.101"
          },
          "doc_count" : 14
        }
      ]
    }
  }
}

3.2 `histogram`

普通的histogram聚合

GET kibana_sample_data_logs/_search
{
  "size": 0,
  "aggs": {
    "aggs1": {
      "histogram": {
        "field": "bytes",
        "interval": 5000
      }
    }
  }
}

{
  "took" : 2,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "aggs1" : {
      "buckets" : [
        {
          "key" : 0.0,
          "doc_count" : 6377
        },
        {
          "key" : 5000.0,
          "doc_count" : 6995
        },
        {
          "key" : 10000.0,
          "doc_count" : 375
        },
        {
          "key" : 15000.0,
          "doc_count" : 327
        }
      ]
    }
  }
}

composite聚合中的histogram

GET kibana_sample_data_logs/_search
{
  "size": 0,
  "aggs": {
    "aggs1": {
      "composite": {
        "sources": [
          {
            "bytesAggs": {
              "histogram": {
                "field": "bytes",
                "interval": 5000
              }
            }
          }
        ]
      }
    }
  }
}

{
  "took" : 19,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "aggs1" : {
      "after_key" : {
        "bytesAggs" : 15000.0
      },
      "buckets" : [
        {
          "key" : {
            "bytesAggs" : 0.0
          },
          "doc_count" : 6377
        },
        {
          "key" : {
            "bytesAggs" : 5000.0
          },
          "doc_count" : 6995
        },
        {
          "key" : {
            "bytesAggs" : 10000.0
          },
          "doc_count" : 375
        },
        {
          "key" : {
            "bytesAggs" : 15000.0
          },
          "doc_count" : 327
        }
      ]
    }
  }
}

3.3 `date_histogram`

普通的时间聚合

GET kibana_sample_data_logs/_search
{
  "size": 0,
  "aggs": {
    "aggs1": {
      "date_histogram": {
        "field": "timestamp",
        "interval": "1M"
      }
    }
  }
}

{
  "took" : 5,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "aggs1" : {
      "buckets" : [
        {
          "key_as_string" : "2021-05-01T00:00:00.000Z",
          "key" : 1619827200000,
          "doc_count" : 6926
        },
        {
          "key_as_string" : "2021-06-01T00:00:00.000Z",
          "key" : 1622505600000,
          "doc_count" : 6943
        },
        {
          "key_as_string" : "2021-07-01T00:00:00.000Z",
          "key" : 1625097600000,
          "doc_count" : 205
        }
      ]
    }
  }
}

composite中的date_histogram

GET kibana_sample_data_logs/_search
{
  "size": 0,
  "aggs": {
    "aggs1": {
      "composite": {
        "sources": [
          {
            "dateAggs": {
              "date_histogram": {
                "field": "timestamp",
                "interval": "1M"
              }
            }
          }
        ]
      }
    }
  }
}

{
  "took" : 28,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "aggs1" : {
      "after_key" : {
        "dateAggs" : 1625097600000
      },
      "buckets" : [
        {
          "key" : {
            "dateAggs" : 1619827200000
          },
          "doc_count" : 6926
        },
        {
          "key" : {
            "dateAggs" : 1622505600000
          },
          "doc_count" : 6943
        },
        {
          "key" : {
            "dateAggs" : 1625097600000
          },
          "doc_count" : 205
        }
      ]
    }
  }
}

3.4 地理位置

3.5 多种混合

普通的混合

GET kibana_sample_data_logs/_search
{
  "size": 0,
  "aggs": {
    "aggs1": {
      "terms": {
        "field": "clientip",
        "size": 3
      }
    },
    "aggs2": {
      "date_histogram": {
        "field": "timestamp",
        "interval": "month"
      }
    }
  }
}

{
  "took" : 2,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "aggs2" : {
      "buckets" : [
        {
          "key_as_string" : "2021-05-01T00:00:00.000Z",
          "key" : 1619827200000,
          "doc_count" : 6926
        },
        {
          "key_as_string" : "2021-06-01T00:00:00.000Z",
          "key" : 1622505600000,
          "doc_count" : 6943
        },
        {
          "key_as_string" : "2021-07-01T00:00:00.000Z",
          "key" : 1625097600000,
          "doc_count" : 205
        }
      ]
    },
    "aggs1" : {
      "doc_count_error_upper_bound" : 0,
      "sum_other_doc_count" : 13919,
      "buckets" : [
        {
          "key" : "30.156.16.164",
          "doc_count" : 100
        },
        {
          "key" : "164.85.94.243",
          "doc_count" : 29
        },
        {
          "key" : "50.184.59.162",
          "doc_count" : 26
        }
      ]
    }
  }
}

composite中的混合source

GET kibana_sample_data_logs/_search
{
  "size": 0,
  "aggs": {
    "aggs1": {
      "composite": {
        "size": 3, 
        "sources": [
          {
            "clientipAggs": {
              "terms": {
                "field": "clientip"
              }
            }
          },
          {
            "dateAggs": {
              "date_histogram": {
                "field": "timestamp",
                "interval": "month"
              }
            }
          }
        ]
      }
    }
  }
}


{
  "took" : 6,
  "timed_out" : false,
  "_shards" : {
    "total" : 1,
    "successful" : 1,
    "skipped" : 0,
    "failed" : 0
  },
  "hits" : {
    "total" : {
      "value" : 10000,
      "relation" : "gte"
    },
    "max_score" : null,
    "hits" : [ ]
  },
  "aggregations" : {
    "aggs1" : {
      "after_key" : {
        "clientipAggs" : "0.207.229.147",
        "dateAggs" : 1619827200000
      },
      "buckets" : [
        {
          "key" : {
            "clientipAggs" : "0.72.176.46",
            "dateAggs" : 1619827200000
          },
          "doc_count" : 6
        },
        {
          "key" : {
            "clientipAggs" : "0.72.176.46",
            "dateAggs" : 1622505600000
          },
          "doc_count" : 8
        },
        {
          "key" : {
            "clientipAggs" : "0.207.229.147",
            "dateAggs" : 1619827200000
          },
          "doc_count" : 6
        }
      ]
    }
  }
}

四、排序

4.1 `source`只有一个元素

如上，如果sources参数只有一个terms: clientip, 排序只能是对应的field: clientip的顺序

注意: 无法根据terms: clientip的_count来排序

4.2 `source`有多个元素

如果sources参数有多个元素，例如:

terms: clientip 和 date_histogram: timestamp, 排序为先按照field: clientip, 然后按照field: timestamp排序

五、分页

使用参数, size 和after_key 实现分页功能

六、子聚合

同普通的bucket聚合一样，composite聚合可以包含子聚合

七、pipeline 聚合

目前, composite聚合和pipeline聚合不兼容，无法一起使用

八、提前优化

可以在索引阶段指定sort, 查询时候排序将更有效

以上是关于elasticsearch基础—— 复合聚合的主要内容，如果未能解决你的问题，请参考以下文章

具有 after_key 和 size 的 ElasticSearch 复合聚合

Elasticsearch全文检索技术一篇文章即可从入门到精通（Elasticsearch安装，安装kibana，安装ik分词器，数据的增删改查，全文检索查询，聚合aggregations）(代码片

ElasticSearch实战（十三）-复合数据字段类型

ElasticSearch实战-复合数据类型

Elasticsearch操作