Elasticsearch Notes

发表于 分类于 ， Disqus： 本文字数： 1.4k 阅读时长 ≈ 5 分钟

Environment

  • Elasticsearch version: 7.x

  • Go version: >= 1.20 (下面示例中使用了 Go Generic 的工具类库 lo、conc … )

    Go Elastic Client: olivere/elastic

    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    package main

    import (
      "context"
      "fmt"

      "github.com/olivere/elastic/v7"
    )

    func main() {
      var (
        host     = []string{"http://192.168.16.28:9200"}
        user     = "elastic"
        password = "*****"
      )

      cli, err := NewClient(host, user, password)
      if err != nil {
        panic(err)
      }

      healthResponse, err := cli.CatHealth().Do(context.Background())
      if err != nil {
        panic(err)
      }

      fmt.Println(healthResponse) // [{1688526238 03:03:58 flower-es green 3 3 2135 1068 0 0 0 0 - 100.0%}]
    }

    func NewClient(host []string, user, password string) (*elastic.Client, error) {
      // 当 es 服务器监听（publish_address）使用内网服务器 ip，而访问（bound_addresses）使用外网IP时，不要设置 client.transport.sniff 为 true。
      // 不设置 client.transport.sniff 时，默认为 false (关闭客户端去嗅探整个集群的状态)。因为在自动发现时会使用内网 IP 进行通信，
      // 导致无法连接到 es 服务器。因此此时需要直接使用 addTransportAddress 方法把集群中其它机器的 ip 地址加到客户端中。
      return elastic.NewClient(elastic.SetSniff(false), elastic.SetURL(host...), elastic.SetBasicAuth(user, password))
    }

Respository

https://github.com/ryan961/es-examples

Examples

Terms Aggregation

Terms aggregation | Elasticsearch Guide [7.17] | Elastic

Kibana Dev Tools

1
2
3
// TestTerms
GET flower-business-*/_search
{"aggs":{"aggs":{"terms":{"field":"platform","order":[{"_count":"desc"}],"size":10}}},"query":{"bool":{"filter":[{"query_string":{"query":"*"}},{"range":{"@timestamp":{"format":"epoch_second","from":1688313600,"include_lower":true,"include_upper":false,"to":1689004799}}}]}},"size":0}
1
2
3
// TestTerms_SubAggregation
GET flower-business-*/_search
{"aggs":{"aggs":{"aggregations":{"platform":{"terms":{"field":"platform"}}},"terms":{"field":"appVersion","order":[{"_count":"desc"}],"size":10}}},"query":{"bool":{"filter":[{"query_string":{"query":"*"}},{"range":{"@timestamp":{"format":"epoch_second","from":1688313600,"include_lower":true,"include_upper":false,"to":1689004799}}}]}},"size":0}

Go Examples

https://github.com/ryan961/es-examples/blob/main/examples/search_aggs_terms_test.go

DateHistogram Aggregation

Date histogram aggregation | Elasticsearch Guide [7.17] | Elastic

Kibana Dev Tools

1
2
3
// TestDateHistogram
GET flower-business-*/_search
{"aggs":{"aggs":{"date_histogram":{"calendar_interval":"1d","field":"@timestamp","time_zone":"Asia/Shanghai"}}},"query":{"bool":{"filter":[{"query_string":{"query":"*"}},{"range":{"@timestamp":{"format":"epoch_second","from":1688313600,"include_lower":true,"include_upper":false,"to":1689004799}}}]}},"size":0}
1
2
3
// TestDateHistogram_SubAggregation
GET flower-business-*/_search
{"aggs":{"aggs":{"aggregations":{"dau":{"cardinality":{"field":"uuid"}}},"date_histogram":{"calendar_interval":"1d","field":"@timestamp","time_zone":"Asia/Shanghai"}}},"query":{"bool":{"filter":[{"query_string":{"query":"*"}},{"range":{"@timestamp":{"format":"epoch_second","from":1688313600,"include_lower":true,"include_upper":false,"to":1689004799}}}]}},"size":0}
1
2
3
// TestDateHistogram_RuntimeMappings
GET flower-business-*/_search
{"aggs":{"aggs":{"aggregations":{"regDay":{"aggregations":{"dau":{"cardinality":{"field":"uuid"}}},"date_histogram":{"calendar_interval":"1d","field":"register_time_date","time_zone":"Asia/Shanghai"}}},"date_histogram":{"calendar_interval":"1d","field":"@timestamp","time_zone":"Asia/Shanghai"}}},"query":{"bool":{"filter":[{"query_string":{"query":"_exists_:register_time"}},{"range":{"@timestamp":{"format":"epoch_second","from":1688313600,"include_lower":true,"include_upper":false,"to":1689004799}}}]}},"runtime_mappings":{"register_time_date":{"script":{"source":"if(doc['register_time'].size()!=0) {emit(doc['register_time'].value*1000);} else {emit(0);}"},"type":"date"}},"size":0}

Go Examples

https://github.com/ryan961/es-examples/blob/main/examples/search_aggs_dateHistogram_test.go

Filters Aggregation

Filters aggregation | Elasticsearch Guide [7.17] | Elastic

Kibana Dev Tools

1
2
3
// TestFilters
GET flower-business-*/_search
{"aggs":{"aggs":{"filters":{"filters":{"itemsLog":{"query_string":{"query":"LogType:itemsLog"}},"loginLog":{"query_string":{"query":"LogType:loginLog"}},"scoreLog":{"query_string":{"query":"LogType:scoreLog"}}}}}},"query":{"bool":{"filter":[{"query_string":{"query":"*"}},{"range":{"@timestamp":{"format":"epoch_second","from":1688313600,"include_lower":true,"include_upper":false,"to":1689004799}}}]}},"size":0}
1
2
3
// TestFilters_SubAggregation
GET flower-business-*/_search
{"aggs":{"aggs":{"aggregations":{"count":{"cardinality":{"field":"uuid"}}},"filters":{"filters":{"itemsLog":{"query_string":{"query":"LogType:itemsLog"}},"loginLog":{"query_string":{"query":"LogType:loginLog"}},"scoreLog":{"query_string":{"query":"LogType:scoreLog"}}}}}},"query":{"bool":{"filter":[{"query_string":{"query":"*"}},{"range":{"@timestamp":{"format":"epoch_second","from":1688313600,"include_lower":true,"include_upper":false,"to":1689004799}}}]}},"size":0}

Go Examples

https://github.com/ryan961/es-examples/blob/main/examples/search_aggs_filters_test.go

Composite Aggregation

Composite aggregation | Elasticsearch Guide [7.17] | Elastic

Kibana Dev Tools

1
2
3
// TestComposite
GET flower-business-*/_search
{"aggs":{"aggs":{"composite":{"size":10000,"sources":[{"customerId":{"terms":{"field":"customerId"}}}]}}},"query":{"bool":{"filter":[{"query_string":{"query":"*"}},{"range":{"@timestamp":{"format":"epoch_second","from":1688054400,"include_lower":true,"include_upper":false,"to":1688745599}}}]}},"size":0}
1
2
3
// TestComposite_SubAggregation
GET flower-business-*/_search
{"aggs":{"aggs":{"aggregations":{"lastLevel":{"max":{"field":"topLevel"}},"lastLoginIn":{"max":{"field":"actTime"}}},"composite":{"size":10000,"sources":[{"customerId":{"terms":{"field":"customerId"}}}]}}},"query":{"bool":{"filter":[{"query_string":{"query":"*"}},{"range":{"@timestamp":{"format":"epoch_second","from":1688054400,"include_lower":true,"include_upper":false,"to":1688745599}}}]}},"size":0}
1
2
3
// TestComposite_DateHistogram
GET flower-business-*/_search
{"aggs":{"aggs":{"composite":{"size":10000,"sources":[{"actDay":{"date_histogram":{"calendar_interval":"1d","field":"@timestamp","time_zone":"Asia/Shanghai"}}},{"customerId":{"terms":{"field":"customerId"}}}]}}},"query":{"bool":{"filter":[{"query_string":{"query":"*"}},{"range":{"@timestamp":{"format":"epoch_second","from":1688054400,"include_lower":true,"include_upper":false,"to":1688745599}}}]}},"size":0}
1
2
3
// TestComposite_DateHistogram_RuntimeMappings
GET flower-business-*/_search
{"runtime_mappings":{"register_time_date":{"type":"date","script":{"source":"if(doc['register_time'].size()!=0)\n        {emit(doc['register_time'].value*1000);}\n        else{emit(0);}"}}},"aggs":{"aggs":{"composite":{"size":10000,"sources":[{"actDay":{"date_histogram":{"calendar_interval":"1d","field":"@timestamp","time_zone":"Asia/Shanghai"}}},{"regDay":{"date_histogram":{"calendar_interval":"1d","field":"register_time_date","time_zone":"Asia/Shanghai"}}},{"customerId":{"terms":{"field":"customerId"}}},{"register_time":{"terms":{"field":"register_time"}}}]}}},"query":{"bool":{"filter":[{"query_string":{"query":"*"}},{"range":{"@timestamp":{"format":"epoch_second","from":1687968000,"include_lower":true,"include_upper":false,"to":1688659199}}}]}},"size":0}
1
2
3
// TestComposite_DateHistogram_Filters
GET flower-business-*/_search
{"aggs":{"aggs":{"aggregations":{"dau":{"aggregations":{"count":{"cardinality":{"field":"uuid"}}},"filter":{"query_string":{"query":"LogType:loginLog"}}},"itemNumbers":{"aggregations":{"sum":{"sum":{"field":"number"}}},"filter":{"query_string":{"query":"LogType:itemsLog"}}},"levelCount":{"filter":{"query_string":{"query":"LogType:scoreLog"}}}},"composite":{"size":10000,"sources":[{"actDay":{"date_histogram":{"calendar_interval":"1d","field":"@timestamp","time_zone":"Asia/Shanghai"}}}]}}},"query":{"bool":{"filter":[{"query_string":{"query":"*"}},{"range":{"@timestamp":{"format":"epoch_second","from":1688054400,"include_lower":true,"include_upper":false,"to":1688745599}}}]}},"size":0}
1
2
3
// TestComposite_DateHistogram_Range
GET flower-business-*/_search
{"aggs":{"aggs":{"aggregations":{"level":{"range":{"field":"topLevel","ranges":[{"from":0,"key":"0~500","to":500},{"from":501,"key":"501~"}]}}},"composite":{"size":10000,"sources":[{"actDay":{"date_histogram":{"calendar_interval":"1d","field":"@timestamp","time_zone":"Asia/Shanghai"}}}]}}},"query":{"bool":{"filter":[{"query_string":{"query":"LogType:loginLog"}},{"range":{"@timestamp":{"format":"epoch_second","from":1688054400,"include_lower":true,"include_upper":false,"to":1688745599}}}]}},"size":0}

Go Examples

https://github.com/ryan961/es-examples/blob/main/examples/search_aggs_composite_test.go

Update By Query

Update By Query API | Elasticsearch Guide [7.17] | Elastic

Kibana Dev Tools

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
POST flower-calcmeta/_update_by_query?ignore_unavailable=true&refresh=true
{
  "query": {
    "query_string": {
      "query": "ad_id:6717889215 AND register_time:1677600000"
    }
  },
  "script": {
    "lang": "painless",
    "source": """ctx._source["show"]=params["show"];ctx._source["active"]=params["active"];ctx._source["register"]=params["register"];ctx._source["register_active"]=params["register_active"];ctx._source["cost_active"]=params["cost_active"];ctx._source["cost_register"]=params["cost_register"];""",
    "params": {
      "show": 3914,
      "active": 6,
      "register": 1,
      "register_active": 0.1667,
      "cost_active": 5.53,
      "cost_register": 33.18
    }
  }
}

Go Examples

https://github.com/ryan961/es-examples/blob/main/gdt_fix_active/main.go

Delete By Query

Delete by query API | Elasticsearch Guide [7.17] | Elastic

Kibana Dev Tools

1
2
3
4
5
6
7
8
POST flower-calcmeta/_delete_by_query
{
  "query": {
    "query_string": {
      "query": "ad_id:6717889215 AND register_time:1677600000"
    }
  }
}

Go Examples

https://github.com/ryan961/es-examples/blob/main/examples/deleteByQuery_test.go

Scroll

Scroll API | Elasticsearch Guide [7.17] | Elastic

Go Examples

https://github.com/ryan961/es-examples/blob/main/examples/scroll_test.go

Script Fields

Retrieve selected fields from a search | Elasticsearch Guide [7.17] | Elastic

Example: Use scripted fields in Kibana to add day_x (days since unlock) field for unlock metrics.
Add path: Stack Management > Index Patterns > flower-business-* > Scripted fields > Add scripted field

1
2
3
4
5
6
7
8
if (doc["msg.code"].size() != 0 && doc["msg.code"].value == 1040 && doc["msg.eventName"].value == "unlockInstanceMap"){
  def now_ts = new Date().getTime();
  def now_inst = Instant.ofEpochMilli(now_ts);
  ZonedDateTime now = ZonedDateTime.ofInstant(now_inst,ZoneId.of('Z'));
  ZonedDateTime lockTime = ZonedDateTime.ofInstant(doc['@timestamp'].value.toInstant(), ZoneId.of('Z'));
  def day = ChronoUnit.DAYS.between(lockTime, now);
  return day
}

Kibana Dev Tools

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
POST test/_doc
{
    "msg":{
        "actionType":"compositeMerge",
        "code":1040,
        "eventName":"merge"
    },
    "islandPassData":{
        "activityId":1,
        "realOpenTime":1672023242702
    },
    "deviceId":"00000000-23c8-1dd0-0000-00005510f0e3",
    "uuid":"4053c58e-7849-11ed-aa83-02734471da14",
    "registerTime":1670649153,
    "unlockSkinAt":1671368118,
    "customerId":25851790,
    "@timestamp":"2023-03-07T21:09:03.000Z"
}

GET test/_doc/mHBV3oYBGotJ2rqXyozh
{}

GET test/_search
{
  "query": {
    "match_all": {}
  },
  "script_fields": {
    "day_x": {
      "script": {
        "lang": "painless",
        "source": """
        def now_ts = new Date().getTime();
        def now_inst = Instant.ofEpochMilli(now_ts);
        ZonedDateTime now = ZonedDateTime.ofInstant(now_inst,ZoneId.of('Z'));
        ZonedDateTime lockTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(doc['islandPassData.realOpenTime'].value), ZoneId.of('Z'));
        def day = ChronoUnit.DAYS.between(lockTime, now);
        return day
        """
      }
    }
  }
}

Script Functions

1
2
3
4
5
6
7
8
9
// 将时间戳(s)转化成 ZonedDateTime 类型
ZonedDateTime lockTime = ZonedDateTime.ofInstant(Instant.ofEpochSecond(doc['unlockSkinAt'].value), ZoneId.of('Z'));

// ms
ZonedDateTime lockTime = ZonedDateTime.ofInstant(Instant.ofEpochMilli(doc['islandPassData.realOpenTime'].value), ZoneId.of('Z'));

ZonedDateTime lockTime = ZonedDateTime.ofInstant(doc['@timestamp'].value.toInstant(), ZoneId.of('Z'));

ZonedDateTime now = ZonedDateTime.ofInstant(Instant.now().toEpochMilli(),ZoneId.of('Z'));

More to come… :)