fix curator

-the old curator does not support ES 5.x
-include curator 5.1.1 and pin version to exactly that to avoid surprises with disruptive updates
-configs reside in /etc/tpot/curator
-will be started daily through /etc/crontab
-by default all logstash indices older than 90 days will be deleted

installer/etc/tpot/curator/actions.yml

@@ -0,0 +1,26 @@
+# Remember, leave a key empty if there is no value.  None will be a string,
+# not a Python "NoneType"
+#
+# Also remember that all examples have 'disable_action' set to True.  If you
+# want to use this action as a template, be sure to set this to False after
+# copying it.
+actions:
+  1:
+    action: delete_indices
+    description: >-
+      Delete indices older than 90 days (based on index name), for logstash-
+      prefixed indices. Ignore the error if the filter does not result in an
+      actionable list of indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: prefix
+      value: logstash-
+    - filtertype: age
+      source: name
+      direction: older
+      timestring: '%Y.%m.%d'
+      unit: days
+      unit_count: 90

installer/etc/tpot/curator/curator.yml

@@ -0,0 +1,21 @@
+# Remember, leave a key empty if there is no value.  None will be a string,
+# not a Python "NoneType"
+client:
+  hosts:
+    - 127.0.0.1
+  port: 64298
+  url_prefix:
+  use_ssl: False
+  certificate:
+  client_cert:
+  client_key:
+  ssl_no_validate: False
+  http_auth:
+  timeout: 30
+  master_only: False
+
+logging:
+  loglevel: INFO
+  logfile: /var/log/curator.log
+  logformat: default
+  blacklist: ['elasticsearch', 'urllib3']

installer/install.sh

@@ -329,6 +329,7 @@ apt-get autoremove -y 2>&1 | dialog --title "[ Pulling updates ]" $myPROGRESSBOX
 # Installing docker-compose, wetty, ctop, elasticdump
 pip install --upgrade pip 2>&1 | dialog --title "[ Installing pip ]" $myPROGRESSBOXCONF
 pip install docker-compose==1.12.0 2>&1 | dialog --title "[ Installing docker-compose ]" $myPROGRESSBOXCONF
+pip install elasticsearch-curator==5.1.1 2>&1 | dialog --title "[ Installing elasticsearch-curator ]" $myPROGRESSBOXCONF
 ln -s /usr/bin/nodejs /usr/bin/node 2>&1 | dialog --title "[ Installing wetty ]" $myPROGRESSBOXCONF
 npm install https://github.com/t3chn0m4g3/wetty -g 2>&1 | dialog --title "[ Installing wetty ]" $myPROGRESSBOXCONF
 npm install https://github.com/t3chn0m4g3/elasticsearch-dump -g 2>&1 | dialog --title "[ Installing elasticsearch-dump ]" $myPROGRESSBOXCONF
@@ -424,8 +425,8 @@ tee -a /etc/crontab 2>&1>/dev/null <<EOF
 # Check if updated images are available and download them
 27 1 * * *      root    /usr/bin/docker-compose -f /etc/tpot/tpot.yml pull
 
-# Delete elastic indices older than 90 days (kibana index is omitted by default)
+# Delete elasticsearch logstash indices older than 90 days
-#27 4 * * *     root    docker exec elk bash -c '/usr/local/bin/curator --host 127.0.0.1 delete indices --older-than 90 --time-unit days --timestring \%Y.\%m.\%d'
+27 4 * * *      root    /usr/local/bin/curator --config /etc/tpot/curator/curator.yml /etc/tpot/curator/actions.yml
 
 # Uploaded binaries are not supposed to be downloaded
 */1 * * * *     root    mv --backup=numbered /data/dionaea/roots/ftp/* /data/dionaea/binaries/