Elasticsearch adalah sebuah basis data yang diciptakan untuk menyimpan dokumen dalam bentuk json, dan fokus pada realtime indexing. Beberapa bulan yang lalu, sempat mencoba-coba untuk menginstall ELK stack (elasticsearch, logstash, kibana) dengan panduan yang ada di sini. Sampai akhirnya berhasil install dengan tampilan Kibana kurang lebih seperti berikut:

• Gambar hanya rekayasa. Gambar asli sudah tidak ada dikarenakan semua log sudah dihapus.

Ya, Kibana dapat memberikan visualisasi dengan sangat baik pada data-data yang disimpan di elasticsearch. Namun karena fungsi utama dari elasticsearch adalah memvisualisasikan data yang besar, maka tak dapat dihindari bahwa space pada disk yang dipakai pun besar. Seperti pada saat dicek di bawah ini.

[root@ns1 ~]# curl -XGET 'http://localhost:9200/_cluster/stats?human&pretty'
{
  "timestamp" : 1467291737491,
  "cluster_name" : "elasticsearch",
  "status" : "yellow",
  "indices" : {
    "count" : 338,
    "shards" : {
      "total" : 1686,
      "primaries" : 1686,
      "replication" : 0.0,
      "index" : {
        "shards" : {
          "min" : 1,
          "max" : 5,
          "avg" : 4.988165680473373
        },
        "primaries" : {
          "min" : 1,
          "max" : 5,
          "avg" : 4.988165680473373
        },
        "replication" : {
          "min" : 0.0,
          "max" : 0.0,
          "avg" : 0.0
        }
      }
    },
    "docs" : {
      "count" : 195222619,
      "deleted" : 0
    },
    "store" : {
      "size" : "47gb",
      "size_in_bytes" : 50527949332,
      "throttle_time" : "0s",
      "throttle_time_in_millis" : 0
    },
    "fielddata" : {
      "memory_size" : "62.5kb",
      "memory_size_in_bytes" : 64040,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size" : "535.8kb",
      "memory_size_in_bytes" : 548672,
      "total_count" : 5130838,
      "hit_count" : 718,
      "miss_count" : 5130120,
      "cache_size" : 139,
      "cache_count" : 201,
      "evictions" : 62
    },
    "completion" : {
      "size" : "0b",
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 11173,
      "memory" : "541.1mb",
      "memory_in_bytes" : 567468090,
      "terms_memory" : "337.4mb",
      "terms_memory_in_bytes" : 353798310,
      "stored_fields_memory" : "18.7mb",
      "stored_fields_memory_in_bytes" : 19647016,
      "term_vectors_memory" : "0b",
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory" : "5.5mb",
      "norms_memory_in_bytes" : 5781952,
      "doc_values_memory" : "179.5mb",
      "doc_values_memory_in_bytes" : 188240812,
      "index_writer_memory" : "0b",
      "index_writer_memory_in_bytes" : 0,
      "index_writer_max_memory" : "1.9gb",
      "index_writer_max_memory_in_bytes" : 2137069155,
      "version_map_memory" : "0b",
      "version_map_memory_in_bytes" : 0,
      "fixed_bit_set" : "0b",
      "fixed_bit_set_memory_in_bytes" : 0
    },
    "percolate" : {
      "total" : 0,
      "time" : "0s",
      "time_in_millis" : 0,
      "current" : 0,
      "memory_size_in_bytes" : -1,
      "memory_size" : "-1b",
      "queries" : 0
    }
  },
  "nodes" : {
    "count" : {
      "total" : 1,
      "master_only" : 0,
      "data_only" : 0,
      "master_data" : 1,
      "client" : 0
    },
    "versions" : [ "2.3.3" ],
    "os" : {
      "available_processors" : 8,
      "allocated_processors" : 8,
      "mem" : {
        "total" : "2.6gb",
        "total_in_bytes" : 2807103488
      },
      "names" : [ {
        "name" : "Linux",
        "count" : 1
      } ]
    },
    "process" : {
      "cpu" : {
        "percent" : 2
      },
      "open_file_descriptors" : {
        "min" : 21238,
        "max" : 21238,
        "avg" : 21238
      }
    },
    "jvm" : {
      "max_uptime" : "29.7d",
      "max_uptime_in_millis" : 2572035456,
      "versions" : [ {
        "version" : "1.8.0_72",
        "vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
        "vm_version" : "25.72-b15",
        "vm_vendor" : "Oracle Corporation",
        "count" : 1
      } ],
      "mem" : {
        "heap_used" : "4.4gb",
        "heap_used_in_bytes" : 4810868200,
        "heap_max" : "11.9gb",
        "heap_max_in_bytes" : 12815171584
      },
      "threads" : 110
    },
    "fs" : {
      "total" : "97.6gb",
      "total_in_bytes" : 104818769920,
      "free" : "13gb",
      "free_in_bytes" : 14055034880,
      "available" : "8.1gb",
      "available_in_bytes" : 8706879488
    },
    "plugins" : [ ]
  }
}

Pada bagian store, ada space 47 gb yang dipakai. Padahal space total (seperti yang tertera pada fs) hanyalah 97.6 gb. Oleh karena itu, timbul keinginan untuk menghapus data yang ada pada elasticsearch.

Cara menghapus data tersebut cukup dengan satu perintah berikut:

curl -XDELETE 'http://localhost:9200/_all'

Dan hasilnya terlihat pada json di bawah ini

{
  "timestamp" : 1467292083679,
  "cluster_name" : "elasticsearch",
  "status" : "yellow",
  "indices" : {
    "count" : 4,
    "shards" : {
      "total" : 16,
      "primaries" : 16,
      "replication" : 0.0,
      "index" : {
        "shards" : {
          "min" : 1,
          "max" : 5,
          "avg" : 4.0
        },
        "primaries" : {
          "min" : 1,
          "max" : 5,
          "avg" : 4.0
        },
        "replication" : {
          "min" : 0.0,
          "max" : 0.0,
          "avg" : 0.0
        }
      }
    },
    "docs" : {
      "count" : 1914,
      "deleted" : 0
    },
    "store" : {
      "size" : "1.7mb",
      "size_in_bytes" : 1853082,
      "throttle_time" : "0s",
      "throttle_time_in_millis" : 0
    },
    "fielddata" : {
      "memory_size" : "0b",
      "memory_size_in_bytes" : 0,
      "evictions" : 0
    },
    "query_cache" : {
      "memory_size" : "0b",
      "memory_size_in_bytes" : 0,
      "total_count" : 18,
      "hit_count" : 0,
      "miss_count" : 18,
      "cache_size" : 0,
      "cache_count" : 0,
      "evictions" : 0
    },
    "completion" : {
      "size" : "0b",
      "size_in_bytes" : 0
    },
    "segments" : {
      "count" : 56,
      "memory" : "754.5kb",
      "memory_in_bytes" : 772652,
      "terms_memory" : "619.8kb",
      "terms_memory_in_bytes" : 634748,
      "stored_fields_memory" : "17kb",
      "stored_fields_memory_in_bytes" : 17472,
      "term_vectors_memory" : "0b",
      "term_vectors_memory_in_bytes" : 0,
      "norms_memory" : "15.1kb",
      "norms_memory_in_bytes" : 15552,
      "doc_values_memory" : "102.4kb",
      "doc_values_memory_in_bytes" : 104880,
      "index_writer_memory" : "2.7mb",
      "index_writer_memory_in_bytes" : 2852836,
      "index_writer_max_memory" : "1.1gb",
      "index_writer_max_memory_in_bytes" : 1281517152,
      "version_map_memory" : "11.9kb",
      "version_map_memory_in_bytes" : 12190,
      "fixed_bit_set" : "0b",
      "fixed_bit_set_memory_in_bytes" : 0
    },
    "percolate" : {
      "total" : 0,
      "time" : "0s",
      "time_in_millis" : 0,
      "current" : 0,
      "memory_size_in_bytes" : -1,
      "memory_size" : "-1b",
      "queries" : 0
    }
  },
  "nodes" : {
    "count" : {
      "total" : 1,
      "master_only" : 0,
      "data_only" : 0,
      "master_data" : 1,
      "client" : 0
    },
    "versions" : [ "2.3.3" ],
    "os" : {
      "available_processors" : 8,
      "allocated_processors" : 8,
      "mem" : {
        "total" : "6.8gb",
        "total_in_bytes" : 7333371904
      },
      "names" : [ {
        "name" : "Linux",
        "count" : 1
      } ]
    },
    "process" : {
      "cpu" : {
        "percent" : 3
      },
      "open_file_descriptors" : {
        "min" : 313,
        "max" : 313,
        "avg" : 313
      }
    },
    "jvm" : {
      "max_uptime" : "29.7d",
      "max_uptime_in_millis" : 2572382149,
      "versions" : [ {
        "version" : "1.8.0_72",
        "vm_name" : "Java HotSpot(TM) 64-Bit Server VM",
        "vm_version" : "25.72-b15",
        "vm_vendor" : "Oracle Corporation",
        "count" : 1
      } ],
      "mem" : {
        "heap_used" : "4.3gb",
        "heap_used_in_bytes" : 4646757064,
        "heap_max" : "11.9gb",
        "heap_max_in_bytes" : 12815171584
      },
      "threads" : 111
    },
    "fs" : {
      "total" : "97.6gb",
      "total_in_bytes" : 104818769920,
      "free" : "60.8gb",
      "free_in_bytes" : 65299656704,
      "available" : "55.8gb",
      "available_in_bytes" : 59951501312
    },
    "plugins" : [ ]
  }
}

[1] [2]