ollama and other older nc changes

.gitignore

@@ -5,3 +5,5 @@ openvpn-as/
 syncthing/data1/
 #wireguard/
 #ubuntu/
+comfyui
+ollama/ollama-local

calibre-web/.env

@@ -1,3 +1,4 @@
 CALIBRE_WEB_NAME=calibre-web
 CALIBRE_NAME=calibre
 CALIBRE_ROOT=/home/tuffy/docker/clone/Dockers_dev/calibre-web
+# calibre-web: admin/admin123

es/.env

@@ -0,0 +1,29 @@
+# Password for the 'elastic' user (at least 6 characters)
+ELASTIC_PASSWORD=123456
+
+# Password for the 'kibana_system' user (at least 6 characters)
+KIBANA_PASSWORD=a123456
+
+# Version of Elastic products
+STACK_VERSION=8.4.3
+
+# Set the cluster name
+CLUSTER_NAME=docker-cluster
+
+# Set to 'basic' or 'trial' to automatically start the 30-day trial
+LICENSE=basic
+#LICENSE=trial
+
+# Port to expose Elasticsearch HTTP API to the host
+ES_PORT=0.0.0.0:9200
+#ES_PORT=127.0.0.1:9200
+
+# Port to expose Kibana to the host
+KIBANA_PORT=5601
+#KIBANA_PORT=80
+
+# Increase or decrease based on the available host memory (in bytes)
+MEM_LIMIT=1073741824
+
+# Project namespace (defaults to the current folder name if not set)
+#COMPOSE_PROJECT_NAME=myproject

es/config/_default/6/_settings.json

@@ -0,0 +1,214 @@
+{
+  "settings": {
+    "number_of_shards": 1,
+    "index.mapping.total_fields.limit": 2000,
+    "analysis": {
+      "analyzer": {
+        "fscrawler_path": {
+          "tokenizer": "fscrawler_path"
+        }
+      },
+      "tokenizer": {
+        "fscrawler_path": {
+          "type": "path_hierarchy"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "raw_as_text": {
+          "path_match": "meta.raw.*",
+          "mapping": {
+            "type": "text",
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            }
+          }
+        }
+      }
+    ],
+    "properties": {
+      "attachment": {
+        "type": "binary",
+        "doc_values": false
+      },
+      "attributes": {
+        "properties": {
+          "group": {
+            "type": "keyword"
+          },
+          "owner": {
+            "type": "keyword"
+          }
+        }
+      },
+      "content": {
+        "type": "text"
+      },
+      "file": {
+        "properties": {
+          "content_type": {
+            "type": "keyword"
+          },
+          "filename": {
+            "type": "keyword",
+            "store": true
+          },
+          "extension": {
+            "type": "keyword"
+          },
+          "filesize": {
+            "type": "long"
+          },
+          "indexed_chars": {
+            "type": "long"
+          },
+          "indexing_date": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "created": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "last_modified": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "last_accessed": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "checksum": {
+            "type": "keyword"
+          },
+          "url": {
+            "type": "keyword",
+            "index": false
+          }
+        }
+      },
+      "meta": {
+        "properties": {
+          "author": {
+            "type": "text"
+          },
+          "date": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "keywords": {
+            "type": "text"
+          },
+          "title": {
+            "type": "text"
+          },
+          "language": {
+            "type": "keyword"
+          },
+          "format": {
+            "type": "text"
+          },
+          "identifier": {
+            "type": "text"
+          },
+          "contributor": {
+            "type": "text"
+          },
+          "coverage": {
+            "type": "text"
+          },
+          "modifier": {
+            "type": "text"
+          },
+          "creator_tool": {
+            "type": "keyword"
+          },
+          "publisher": {
+            "type": "text"
+          },
+          "relation": {
+            "type": "text"
+          },
+          "rights": {
+            "type": "text"
+          },
+          "source": {
+            "type": "text"
+          },
+          "type": {
+            "type": "text"
+          },
+          "description": {
+            "type": "text"
+          },
+          "created": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "print_date": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "metadata_date": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "latitude": {
+            "type": "text"
+          },
+          "longitude": {
+            "type": "text"
+          },
+          "altitude": {
+            "type": "text"
+          },
+          "rating": {
+            "type": "byte"
+          },
+          "comments": {
+            "type": "text"
+          }
+        }
+      },
+      "path": {
+        "properties": {
+          "real": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          },
+          "root": {
+            "type": "keyword"
+          },
+          "virtual": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

es/config/_default/6/_settings_folder.json

@@ -0,0 +1,64 @@
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "fscrawler_path": {
+          "tokenizer": "fscrawler_path"
+        }
+      },
+      "tokenizer": {
+        "fscrawler_path": {
+          "type": "path_hierarchy"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "properties" : {
+      "file": {
+        "properties": {
+          "content_type": {
+            "type": "keyword"
+          },
+          "filename": {
+            "type": "keyword",
+            "store": true
+          }
+        }
+      },
+      "path": {
+        "properties": {
+          "real": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          },
+          "root": {
+            "type": "keyword"
+          },
+          "virtual": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

es/config/_default/7/_settings.json

@@ -0,0 +1,214 @@
+{
+  "settings": {
+    "number_of_shards": 1,
+    "index.mapping.total_fields.limit": 2000,
+    "analysis": {
+      "analyzer": {
+        "fscrawler_path": {
+          "tokenizer": "fscrawler_path"
+        }
+      },
+      "tokenizer": {
+        "fscrawler_path": {
+          "type": "path_hierarchy"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "raw_as_text": {
+          "path_match": "meta.raw.*",
+          "mapping": {
+            "type": "text",
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            }
+          }
+        }
+      }
+    ],
+    "properties": {
+      "attachment": {
+        "type": "binary",
+        "doc_values": false
+      },
+      "attributes": {
+        "properties": {
+          "group": {
+            "type": "keyword"
+          },
+          "owner": {
+            "type": "keyword"
+          }
+        }
+      },
+      "content": {
+        "type": "text"
+      },
+      "file": {
+        "properties": {
+          "content_type": {
+            "type": "keyword"
+          },
+          "filename": {
+            "type": "keyword",
+            "store": true
+          },
+          "extension": {
+            "type": "keyword"
+          },
+          "filesize": {
+            "type": "long"
+          },
+          "indexed_chars": {
+            "type": "long"
+          },
+          "indexing_date": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "created": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "last_modified": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "last_accessed": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "checksum": {
+            "type": "keyword"
+          },
+          "url": {
+            "type": "keyword",
+            "index": false
+          }
+        }
+      },
+      "meta": {
+        "properties": {
+          "author": {
+            "type": "text"
+          },
+          "date": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "keywords": {
+            "type": "text"
+          },
+          "title": {
+            "type": "text"
+          },
+          "language": {
+            "type": "keyword"
+          },
+          "format": {
+            "type": "text"
+          },
+          "identifier": {
+            "type": "text"
+          },
+          "contributor": {
+            "type": "text"
+          },
+          "coverage": {
+            "type": "text"
+          },
+          "modifier": {
+            "type": "text"
+          },
+          "creator_tool": {
+            "type": "keyword"
+          },
+          "publisher": {
+            "type": "text"
+          },
+          "relation": {
+            "type": "text"
+          },
+          "rights": {
+            "type": "text"
+          },
+          "source": {
+            "type": "text"
+          },
+          "type": {
+            "type": "text"
+          },
+          "description": {
+            "type": "text"
+          },
+          "created": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "print_date": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "metadata_date": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "latitude": {
+            "type": "text"
+          },
+          "longitude": {
+            "type": "text"
+          },
+          "altitude": {
+            "type": "text"
+          },
+          "rating": {
+            "type": "byte"
+          },
+          "comments": {
+            "type": "text"
+          }
+        }
+      },
+      "path": {
+        "properties": {
+          "real": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          },
+          "root": {
+            "type": "keyword"
+          },
+          "virtual": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

es/config/_default/7/_settings_folder.json

@@ -0,0 +1,64 @@
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "fscrawler_path": {
+          "tokenizer": "fscrawler_path"
+        }
+      },
+      "tokenizer": {
+        "fscrawler_path": {
+          "type": "path_hierarchy"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "properties" : {
+      "file": {
+        "properties": {
+          "content_type": {
+            "type": "keyword"
+          },
+          "filename": {
+            "type": "keyword",
+            "store": true
+          }
+        }
+      },
+      "path": {
+        "properties": {
+          "real": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          },
+          "root": {
+            "type": "keyword"
+          },
+          "virtual": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

es/config/_default/7/_wpsearch_settings.json

@@ -0,0 +1,109 @@
+{
+  "name": "SOURCE_NAME",
+  "context": "organization",
+  "schema": {
+    "body": "text",
+    "comments": "text",
+    "tags": "text",
+    "title": "text",
+    "type": "text",
+    "url": "text",
+    "extension": "text",
+    "mime_type": "text",
+    "path": "text",
+    "size": "number",
+    "created_by": "text",
+    "name": "text",
+    "language": "text",
+    "text_size": "number",
+    "created_at": "date",
+    "last_modified": "date"
+  },
+  "display": {
+    "title_field": "title",
+    "subtitle_field": "name",
+    "description_field": "body",
+    "url_field": "url",
+    "media_type_field": "mime_type",
+    "created_by_field": "created_by",
+    "detail_fields": [
+      {
+        "field_name": "created_by",
+        "label": "Author"
+      },
+      {
+        "field_name": "tags",
+        "label": "Tags"
+      },
+      {
+        "field_name": "language",
+        "label": "Language"
+      },
+      {
+        "field_name": "last_modified",
+        "label": "Last Modification Date"
+      },
+      {
+        "field_name": "created_at",
+        "label": "Creation date"
+      },
+      {
+        "field_name": "comments",
+        "label": "Comments"
+      },
+      {
+        "field_name": "extension",
+        "label": "Extension"
+      },
+      {
+        "field_name": "size",
+        "label": "File size"
+      },
+      {
+        "field_name": "text_size",
+        "label": "Extracted text size"
+      },
+      {
+        "field_name": "path",
+        "label": "Path"
+      },
+      {
+        "field_name": "body",
+        "label": "Content"
+      }
+    ],
+    "color": "#000000"
+  },
+  "facets":
+  {
+    "overrides":
+    [
+      {
+        "display_name": "Media Type",
+        "field": "mime_type",
+        "enabled": true
+      },
+      {
+        "display_name": "Extension",
+        "field": "extension",
+        "enabled": true
+      },
+      {
+        "display_name": "Tags",
+        "field": "tags",
+        "enabled": true
+      },
+      {
+        "display_name": "Created By",
+        "field": "created_by",
+        "enabled": true
+      },
+      {
+        "display_name": "Language",
+        "field": "language",
+        "enabled": true
+      }
+    ]
+  },
+  "is_searchable": true
+}

es/config/_default/8/_settings.json

@@ -0,0 +1,214 @@
+{
+  "settings": {
+    "number_of_shards": 1,
+    "index.mapping.total_fields.limit": 2000,
+    "analysis": {
+      "analyzer": {
+        "fscrawler_path": {
+          "tokenizer": "fscrawler_path"
+        }
+      },
+      "tokenizer": {
+        "fscrawler_path": {
+          "type": "path_hierarchy"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "raw_as_text": {
+          "path_match": "meta.raw.*",
+          "mapping": {
+            "type": "text",
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            }
+          }
+        }
+      }
+    ],
+    "properties": {
+      "attachment": {
+        "type": "binary",
+        "doc_values": false
+      },
+      "attributes": {
+        "properties": {
+          "group": {
+            "type": "keyword"
+          },
+          "owner": {
+            "type": "keyword"
+          }
+        }
+      },
+      "content": {
+        "type": "text"
+      },
+      "file": {
+        "properties": {
+          "content_type": {
+            "type": "keyword"
+          },
+          "filename": {
+            "type": "keyword",
+            "store": true
+          },
+          "extension": {
+            "type": "keyword"
+          },
+          "filesize": {
+            "type": "long"
+          },
+          "indexed_chars": {
+            "type": "long"
+          },
+          "indexing_date": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "created": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "last_modified": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "last_accessed": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "checksum": {
+            "type": "keyword"
+          },
+          "url": {
+            "type": "keyword",
+            "index": false
+          }
+        }
+      },
+      "meta": {
+        "properties": {
+          "author": {
+            "type": "text"
+          },
+          "date": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "keywords": {
+            "type": "text"
+          },
+          "title": {
+            "type": "text"
+          },
+          "language": {
+            "type": "keyword"
+          },
+          "format": {
+            "type": "text"
+          },
+          "identifier": {
+            "type": "text"
+          },
+          "contributor": {
+            "type": "text"
+          },
+          "coverage": {
+            "type": "text"
+          },
+          "modifier": {
+            "type": "text"
+          },
+          "creator_tool": {
+            "type": "keyword"
+          },
+          "publisher": {
+            "type": "text"
+          },
+          "relation": {
+            "type": "text"
+          },
+          "rights": {
+            "type": "text"
+          },
+          "source": {
+            "type": "text"
+          },
+          "type": {
+            "type": "text"
+          },
+          "description": {
+            "type": "text"
+          },
+          "created": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "print_date": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "metadata_date": {
+            "type": "date",
+            "format": "date_optional_time"
+          },
+          "latitude": {
+            "type": "text"
+          },
+          "longitude": {
+            "type": "text"
+          },
+          "altitude": {
+            "type": "text"
+          },
+          "rating": {
+            "type": "byte"
+          },
+          "comments": {
+            "type": "text"
+          }
+        }
+      },
+      "path": {
+        "properties": {
+          "real": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          },
+          "root": {
+            "type": "keyword"
+          },
+          "virtual": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

es/config/_default/8/_settings_folder.json

@@ -0,0 +1,64 @@
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "fscrawler_path": {
+          "tokenizer": "fscrawler_path"
+        }
+      },
+      "tokenizer": {
+        "fscrawler_path": {
+          "type": "path_hierarchy"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "properties" : {
+      "file": {
+        "properties": {
+          "content_type": {
+            "type": "keyword"
+          },
+          "filename": {
+            "type": "keyword",
+            "store": true
+          }
+        }
+      },
+      "path": {
+        "properties": {
+          "real": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          },
+          "root": {
+            "type": "keyword"
+          },
+          "virtual": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

es/config/_default/8/_wpsearch_settings.json

@@ -0,0 +1,109 @@
+{
+  "name": "SOURCE_NAME",
+  "context": "organization",
+  "schema": {
+    "body": "text",
+    "comments": "text",
+    "tags": "text",
+    "title": "text",
+    "type": "text",
+    "url": "text",
+    "extension": "text",
+    "mime_type": "text",
+    "path": "text",
+    "size": "number",
+    "created_by": "text",
+    "name": "text",
+    "language": "text",
+    "text_size": "number",
+    "created_at": "date",
+    "last_modified": "date"
+  },
+  "display": {
+    "title_field": "title",
+    "subtitle_field": "name",
+    "description_field": "body",
+    "url_field": "url",
+    "media_type_field": "mime_type",
+    "created_by_field": "created_by",
+    "detail_fields": [
+      {
+        "field_name": "created_by",
+        "label": "Author"
+      },
+      {
+        "field_name": "tags",
+        "label": "Tags"
+      },
+      {
+        "field_name": "language",
+        "label": "Language"
+      },
+      {
+        "field_name": "last_modified",
+        "label": "Last Modification Date"
+      },
+      {
+        "field_name": "created_at",
+        "label": "Creation date"
+      },
+      {
+        "field_name": "comments",
+        "label": "Comments"
+      },
+      {
+        "field_name": "extension",
+        "label": "Extension"
+      },
+      {
+        "field_name": "size",
+        "label": "File size"
+      },
+      {
+        "field_name": "text_size",
+        "label": "Extracted text size"
+      },
+      {
+        "field_name": "path",
+        "label": "Path"
+      },
+      {
+        "field_name": "body",
+        "label": "Content"
+      }
+    ],
+    "color": "#000000"
+  },
+  "facets":
+  {
+    "overrides":
+    [
+      {
+        "display_name": "Media Type",
+        "field": "mime_type",
+        "enabled": true
+      },
+      {
+        "display_name": "Extension",
+        "field": "extension",
+        "enabled": true
+      },
+      {
+        "display_name": "Tags",
+        "field": "tags",
+        "enabled": true
+      },
+      {
+        "display_name": "Created By",
+        "field": "created_by",
+        "enabled": true
+      },
+      {
+        "display_name": "Language",
+        "field": "language",
+        "enabled": true
+      }
+    ]
+  },
+  "is_searchable": true
+}

es/config/tuffy_es/_settings.yaml

@@ -0,0 +1,41 @@
+---
+name: "tuffy_es"
+fs:
+  url: "/tmp/es"
+  update_rate: "5s"
+  includes:
+  - "*/*"
+  - "*.doc"
+  - "*.pdf"
+  json_support: false
+  filename_as_id: false
+  add_filesize: true
+  remove_deleted: true
+  add_as_inner_object: false
+  store_source: false
+  index_content: true
+  attributes_support: false
+  raw_metadata: true
+  xml_support: false
+  index_folders: true
+  lang_detect: true
+  continue_on_error: true
+  ocr:
+    language: "eng"
+    enabled: true
+    pdf_strategy: "ocr_and_text"
+  follow_symlinks: false
+elasticsearch:
+  nodes:
+    - url: "https://es01:9200"
+    - url: "https://es02:9200"
+    - url: "https://es03:9200"
+  bulk_size: 100
+  flush_interval: "5s"
+  byte_size: "2mb"
+  username: "elastic"
+  password: "123456"
+  ssl_verification: false 
+rest:
+    url: "http://fscrawler:8080"
+

es/docker-compose.yml

@@ -0,0 +1,273 @@
+version: "2.2"
+
+services:
+  setup:
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    networks:
+      - elastic
+    volumes:
+      - certs:/usr/share/elasticsearch/config/certs
+    user: "0"
+    command: >
+      bash -c '
+        if [ x${ELASTIC_PASSWORD} == x ]; then
+          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
+          exit 1;
+        elif [ x${KIBANA_PASSWORD} == x ]; then
+          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
+          exit 1;
+        fi;
+        if [ ! -f config/certs/ca.zip ]; then
+          echo "Creating CA";
+          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
+          unzip config/certs/ca.zip -d config/certs;
+        fi;
+        if [ ! -f config/certs/certs.zip ]; then
+          echo "Creating certs";
+          echo -ne \
+          "instances:\n"\
+          "  - name: es01\n"\
+          "    dns:\n"\
+          "      - es01\n"\
+          "      - localhost\n"\
+          "    ip:\n"\
+          "      - 127.0.0.1\n"\
+          "  - name: es02\n"\
+          "    dns:\n"\
+          "      - es02\n"\
+          "      - localhost\n"\
+          "    ip:\n"\
+          "      - 127.0.0.1\n"\
+          "  - name: es03\n"\
+          "    dns:\n"\
+          "      - es03\n"\
+          "      - localhost\n"\
+          "    ip:\n"\
+          "      - 127.0.0.1\n"\
+          > config/certs/instances.yml;
+          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
+          unzip config/certs/certs.zip -d config/certs;
+        fi;
+        echo "Setting file permissions"
+        chown -R root:root config/certs;
+        find . -type d -exec chmod 750 \{\} \;;
+        find . -type f -exec chmod 640 \{\} \;;
+        echo "Waiting for Elasticsearch availability";
+        until curl -s --cacert config/certs/ca/ca.crt https://es01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
+        echo "Setting kibana_system password";
+        until curl -s -X POST --cacert config/certs/ca/ca.crt -u "elastic:${ELASTIC_PASSWORD}" -H "Content-Type: application/json" https://es01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
+        echo "All done!";
+      '
+    healthcheck:
+      test: ["CMD-SHELL", "[ -f config/certs/es01/es01.crt ]"]
+      interval: 1s
+      timeout: 5s
+      retries: 120
+
+  es01:
+    container_name: es01 
+    networks:
+      - elastic
+    depends_on:
+      setup:
+        condition: service_healthy
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    volumes:
+      - certs:/usr/share/elasticsearch/config/certs
+      - esdata01:/usr/share/elasticsearch/data
+    ports:
+      - ${ES_PORT}:9200
+    environment:
+      - node.name=es01
+      - cluster.name=${CLUSTER_NAME}
+      - cluster.initial_master_nodes=es01,es02,es03
+      - discovery.seed_hosts=es02,es03
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=true
+      - xpack.security.http.ssl.enabled=true
+      - xpack.security.http.ssl.key=certs/es01/es01.key
+      - xpack.security.http.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.http.ssl.verification_mode=certificate
+      - xpack.security.transport.ssl.enabled=true
+      - xpack.security.transport.ssl.key=certs/es01/es01.key
+      - xpack.security.transport.ssl.certificate=certs/es01/es01.crt
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.verification_mode=certificate
+      - xpack.license.self_generated.type=${LICENSE}
+    mem_limit: ${MEM_LIMIT}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+
+
+  es02:
+    container_name: es02 
+    networks:
+      - elastic
+    depends_on:
+      - es01
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    volumes:
+      - certs:/usr/share/elasticsearch/config/certs
+      - esdata02:/usr/share/elasticsearch/data
+    environment:
+      - node.name=es02
+      - cluster.name=${CLUSTER_NAME}
+      - cluster.initial_master_nodes=es01,es02,es03
+      - discovery.seed_hosts=es01,es03
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=true
+      - xpack.security.http.ssl.enabled=true
+      - xpack.security.http.ssl.key=certs/es02/es02.key
+      - xpack.security.http.ssl.certificate=certs/es02/es02.crt
+      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.http.ssl.verification_mode=certificate
+      - xpack.security.transport.ssl.enabled=true
+      - xpack.security.transport.ssl.key=certs/es02/es02.key
+      - xpack.security.transport.ssl.certificate=certs/es02/es02.crt
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.verification_mode=certificate
+      - xpack.license.self_generated.type=${LICENSE}
+    mem_limit: ${MEM_LIMIT}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+
+  es03:
+    container_name: es03 
+    networks:
+      - elastic
+    depends_on:
+      - es02
+    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
+    volumes:
+      - certs:/usr/share/elasticsearch/config/certs
+      - esdata03:/usr/share/elasticsearch/data
+    environment:
+      - node.name=es03
+      - cluster.name=${CLUSTER_NAME}
+      - cluster.initial_master_nodes=es01,es02,es03
+      - discovery.seed_hosts=es01,es02
+      - bootstrap.memory_lock=true
+      - xpack.security.enabled=true
+      - xpack.security.http.ssl.enabled=true
+      - xpack.security.http.ssl.key=certs/es03/es03.key
+      - xpack.security.http.ssl.certificate=certs/es03/es03.crt
+      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.http.ssl.verification_mode=certificate
+      - xpack.security.transport.ssl.enabled=true
+      - xpack.security.transport.ssl.key=certs/es03/es03.key
+      - xpack.security.transport.ssl.certificate=certs/es03/es03.crt
+      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
+      - xpack.security.transport.ssl.verification_mode=certificate
+      - xpack.license.self_generated.type=${LICENSE}
+    mem_limit: ${MEM_LIMIT}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+
+  kibana:
+    networks:
+      - elastic
+    depends_on:
+      es01:
+        condition: service_healthy
+      es02:
+        condition: service_healthy
+      es03:
+        condition: service_healthy
+    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
+    volumes:
+      - certs:/usr/share/kibana/config/certs
+      - kibanadata:/usr/share/kibana/data
+    ports:
+      - ${KIBANA_PORT}:5601
+    environment:
+      - SERVERNAME=kibana
+      - ELASTICSEARCH_HOSTS=https://es01:9200
+      - ELASTICSEARCH_USERNAME=kibana_system
+      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
+      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
+    mem_limit: ${MEM_LIMIT}
+    healthcheck:
+      test:
+        [
+          "CMD-SHELL",
+          "curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",
+        ]
+      interval: 10s
+      timeout: 10s
+      retries: 120
+
+  # FSCrawler
+  fscrawler:
+    image: dadoonet/fscrawler:2.10-SNAPSHOT
+    container_name: fscrawler
+    restart: "no" 
+    volumes:
+      - /home/tuffy/Downloads:/tmp/es:ro
+      - ./config:/root/.fscrawler
+      - ./logs:/usr/share/fscrawler/logs
+#    depends_on:
+#        condition: service_healthy
+#      es02:
+#        condition: service_healthy
+#      es03:
+#        condition: service_healthy
+    ports:
+      - 8200:8080
+    command: fscrawler tuffy_es --restart --rest
+    networks:
+      - elastic
+
+
+volumes:
+  certs:
+    driver: local
+  esdata01:
+    driver: local
+  esdata02:
+    driver: local
+  esdata03:
+    driver: local
+  kibanadata:
+    driver: local
+
+
+networks:
+  elastic:
+    driver: bridge
+
+

es/es.sh

@@ -0,0 +1,2 @@
+curl -k -X GET "https://localhost:9200/inde" -u elastic:123456
+

es/logs/fscrawler.log

@@ -0,0 +1,40 @@
+15:43:01,480 INFO  [f.console] ,----------------------------------------------------------------------------------------------------.
+|       ,---,.  .--.--.     ,----..                                     ,--,           2.10-SNAPSHOT |
+|     ,'  .' | /  /    '.  /   /   \                                  ,--.'|                         |
+|   ,---.'   ||  :  /`. / |   :     :  __  ,-.                   .---.|  | :               __  ,-.   |
+|   |   |   .';  |  |--`  .   |  ;. /,' ,'/ /|                  /. ./|:  : '             ,' ,'/ /|   |
+|   :   :  :  |  :  ;_    .   ; /--` '  | |' | ,--.--.       .-'-. ' ||  ' |      ,---.  '  | |' |   |
+|   :   |  |-, \  \    `. ;   | ;    |  |   ,'/       \     /___/ \: |'  | |     /     \ |  |   ,'   |
+|   |   :  ;/|  `----.   \|   : |    '  :  / .--.  .-. | .-'.. '   ' .|  | :    /    /  |'  :  /     |
+|   |   |   .'  __ \  \  |.   | '___ |  | '   \__\/: . ./___/ \:     ''  : |__ .    ' / ||  | '      |
+|   '   :  '   /  /`--'  /'   ; : .'|;  : |   ," .--.; |.   \  ' .\   |  | '.'|'   ;   /|;  : |      |
+|   |   |  |  '--'.     / '   | '/  :|  , ;  /  /  ,.  | \   \   ' \ |;  :    ;'   |  / ||  , ;      |
+|   |   :  \    `--'---'  |   :    /  ---'  ;  :   .'   \ \   \  |--" |  ,   / |   :    | ---'       |
+|   |   | ,'               \   \ .'         |  ,     .-./  \   \ |     ---`-'   \   \  /             |
+|   `----'                  `---`            `--`---'       '---"                `----'              |
++----------------------------------------------------------------------------------------------------+
+|                                        You know, for Files!                                        |
+|                                     Made from France with Love                                     |
+|                           Source: https://github.com/dadoonet/fscrawler/                           |
+|                          Documentation: https://fscrawler.readthedocs.io/                          |
+`----------------------------------------------------------------------------------------------------'
+
+15:43:01,492 INFO  [f.p.e.c.f.c.BootstrapChecks] Memory [Free/Total=Percent]: HEAP [484.6mb/7.8gb=6.06%], RAM [16.3gb/31.2gb=52.26%], Swap [1.8gb/1.9gb=92.57%].
+15:43:01,653 INFO  [f.p.e.c.f.FsCrawlerImpl] Starting FS crawler
+15:43:01,653 INFO  [f.p.e.c.f.FsCrawlerImpl] FS crawler started in watch mode. It will run unless you stop it with CTRL+C.
+15:43:01,721 WARN  [f.p.e.c.f.c.ElasticsearchClient] We are not doing SSL verification. It's not recommended for production.
+15:43:02,062 INFO  [f.p.e.c.f.c.ElasticsearchClient] Elasticsearch Client connected to a node running version 8.4.3
+15:43:02,065 WARN  [f.p.e.c.f.c.ElasticsearchClient] We are not doing SSL verification. It's not recommended for production.
+15:43:02,106 INFO  [f.p.e.c.f.c.ElasticsearchClient] Elasticsearch Client connected to a node running version 8.4.3
+15:43:02,197 WARN  [f.p.e.c.f.c.ElasticsearchClient] failed to create index [tuffy_es], disabling crawler...
+15:43:02,197 FATAL [f.p.e.c.f.c.FsCrawlerCli] We can not start Elasticsearch Client. Exiting.
+fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClientException: error while creating index tuffy_es: {error={root_cause=[{"type":"security_exception","reason":"unable to authenticate user [elastic] for REST request [\/tuffy_es]","header":{"WWW-Authenticate":["Basic realm=\"security\" charset=\"UTF-8\"","Bearer realm=\"security\"","ApiKey"]}}], type=security_exception, reason=unable to authenticate user [elastic] for REST request [/tuffy_es], header={WWW-Authenticate=["Basic realm=\"security\" charset=\"UTF-8\"","Bearer realm=\"security\"","ApiKey"]}}, status=401}
+	at fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient.createIndex(ElasticsearchClient.java:280) ~[fscrawler-elasticsearch-client-2.10-SNAPSHOT.jar:?]
+	at fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient.createIndex(ElasticsearchClient.java:720) ~[fscrawler-elasticsearch-client-2.10-SNAPSHOT.jar:?]
+	at fr.pilato.elasticsearch.crawler.fs.client.ElasticsearchClient.createIndices(ElasticsearchClient.java:437) ~[fscrawler-elasticsearch-client-2.10-SNAPSHOT.jar:?]
+	at fr.pilato.elasticsearch.crawler.fs.service.FsCrawlerDocumentServiceElasticsearchImpl.createSchema(FsCrawlerDocumentServiceElasticsearchImpl.java:71) ~[fscrawler-core-2.10-SNAPSHOT.jar:?]
+	at fr.pilato.elasticsearch.crawler.fs.FsCrawlerImpl.start(FsCrawlerImpl.java:118) ~[fscrawler-core-2.10-SNAPSHOT.jar:?]
+	at fr.pilato.elasticsearch.crawler.fs.cli.FsCrawlerCli.startEsClient(FsCrawlerCli.java:322) ~[fscrawler-cli-2.10-SNAPSHOT.jar:?]
+	at fr.pilato.elasticsearch.crawler.fs.cli.FsCrawlerCli.main(FsCrawlerCli.java:298) ~[fscrawler-cli-2.10-SNAPSHOT.jar:?]
+15:43:02,204 INFO  [f.p.e.c.f.FsCrawlerImpl] FS crawler [tuffy_es] stopped
+15:43:02,204 INFO  [f.p.e.c.f.FsCrawlerImpl] FS crawler [tuffy_es] stopped

es/vm_max_map.sh

@@ -0,0 +1,2 @@
+sudo sysctl -w vm.max_map_count=262144
+

es_kibana/.env

@@ -0,0 +1 @@
+DATA_VOLUME_ROOT=./data 

es_kibana/config/_default/6/_settings.json

@@ -0,0 +1,214 @@
+{
+  "settings": {
+    "number_of_shards": 1,
+    "index.mapping.total_fields.limit": 2000,
+    "analysis": {
+      "analyzer": {
+        "fscrawler_path": {
+          "tokenizer": "fscrawler_path"
+        }
+      },
+      "tokenizer": {
+        "fscrawler_path": {
+          "type": "path_hierarchy"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "raw_as_text": {
+          "path_match": "meta.raw.*",
+          "mapping": {
+            "type": "text",
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            }
+          }
+        }
+      }
+    ],
+    "properties": {
+      "attachment": {
+        "type": "binary",
+        "doc_values": false
+      },
+      "attributes": {
+        "properties": {
+          "group": {
+            "type": "keyword"
+          },
+          "owner": {
+            "type": "keyword"
+          }
+        }
+      },
+      "content": {
+        "type": "text"
+      },
+      "file": {
+        "properties": {
+          "content_type": {
+            "type": "keyword"
+          },
+          "filename": {
+            "type": "keyword",
+            "store": true
+          },
+          "extension": {
+            "type": "keyword"
+          },
+          "filesize": {
+            "type": "long"
+          },
+          "indexed_chars": {
+            "type": "long"
+          },
+          "indexing_date": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "created": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "last_modified": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "last_accessed": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "checksum": {
+            "type": "keyword"
+          },
+          "url": {
+            "type": "keyword",
+            "index": false
+          }
+        }
+      },
+      "meta": {
+        "properties": {
+          "author": {
+            "type": "text"
+          },
+          "date": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "keywords": {
+            "type": "text"
+          },
+          "title": {
+            "type": "text"
+          },
+          "language": {
+            "type": "keyword"
+          },
+          "format": {
+            "type": "text"
+          },
+          "identifier": {
+            "type": "text"
+          },
+          "contributor": {
+            "type": "text"
+          },
+          "coverage": {
+            "type": "text"
+          },
+          "modifier": {
+            "type": "text"
+          },
+          "creator_tool": {
+            "type": "keyword"
+          },
+          "publisher": {
+            "type": "text"
+          },
+          "relation": {
+            "type": "text"
+          },
+          "rights": {
+            "type": "text"
+          },
+          "source": {
+            "type": "text"
+          },
+          "type": {
+            "type": "text"
+          },
+          "description": {
+            "type": "text"
+          },
+          "created": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "print_date": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "metadata_date": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "latitude": {
+            "type": "text"
+          },
+          "longitude": {
+            "type": "text"
+          },
+          "altitude": {
+            "type": "text"
+          },
+          "rating": {
+            "type": "byte"
+          },
+          "comments": {
+            "type": "text"
+          }
+        }
+      },
+      "path": {
+        "properties": {
+          "real": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          },
+          "root": {
+            "type": "keyword"
+          },
+          "virtual": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

es_kibana/config/_default/6/_settings_folder.json

@@ -0,0 +1,32 @@
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "fscrawler_path": {
+          "tokenizer": "fscrawler_path"
+        }
+      },
+      "tokenizer": {
+        "fscrawler_path": {
+          "type": "path_hierarchy"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "properties" : {
+      "real" : {
+        "type" : "keyword",
+        "store" : true
+      },
+      "root" : {
+        "type" : "keyword",
+        "store" : true
+      },
+      "virtual" : {
+        "type" : "keyword",
+        "store" : true
+      }
+    }
+  }
+}

es_kibana/config/_default/7/_settings.json

@@ -0,0 +1,214 @@
+{
+  "settings": {
+    "number_of_shards": 1,
+    "index.mapping.total_fields.limit": 2000,
+    "analysis": {
+      "analyzer": {
+        "fscrawler_path": {
+          "tokenizer": "fscrawler_path"
+        }
+      },
+      "tokenizer": {
+        "fscrawler_path": {
+          "type": "path_hierarchy"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "raw_as_text": {
+          "path_match": "meta.raw.*",
+          "mapping": {
+            "type": "text",
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            }
+          }
+        }
+      }
+    ],
+    "properties": {
+      "attachment": {
+        "type": "binary",
+        "doc_values": false
+      },
+      "attributes": {
+        "properties": {
+          "group": {
+            "type": "keyword"
+          },
+          "owner": {
+            "type": "keyword"
+          }
+        }
+      },
+      "content": {
+        "type": "text"
+      },
+      "file": {
+        "properties": {
+          "content_type": {
+            "type": "keyword"
+          },
+          "filename": {
+            "type": "keyword",
+            "store": true
+          },
+          "extension": {
+            "type": "keyword"
+          },
+          "filesize": {
+            "type": "long"
+          },
+          "indexed_chars": {
+            "type": "long"
+          },
+          "indexing_date": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "created": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "last_modified": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "last_accessed": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "checksum": {
+            "type": "keyword"
+          },
+          "url": {
+            "type": "keyword",
+            "index": false
+          }
+        }
+      },
+      "meta": {
+        "properties": {
+          "author": {
+            "type": "text"
+          },
+          "date": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "keywords": {
+            "type": "text"
+          },
+          "title": {
+            "type": "text"
+          },
+          "language": {
+            "type": "keyword"
+          },
+          "format": {
+            "type": "text"
+          },
+          "identifier": {
+            "type": "text"
+          },
+          "contributor": {
+            "type": "text"
+          },
+          "coverage": {
+            "type": "text"
+          },
+          "modifier": {
+            "type": "text"
+          },
+          "creator_tool": {
+            "type": "keyword"
+          },
+          "publisher": {
+            "type": "text"
+          },
+          "relation": {
+            "type": "text"
+          },
+          "rights": {
+            "type": "text"
+          },
+          "source": {
+            "type": "text"
+          },
+          "type": {
+            "type": "text"
+          },
+          "description": {
+            "type": "text"
+          },
+          "created": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "print_date": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "metadata_date": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "latitude": {
+            "type": "text"
+          },
+          "longitude": {
+            "type": "text"
+          },
+          "altitude": {
+            "type": "text"
+          },
+          "rating": {
+            "type": "byte"
+          },
+          "comments": {
+            "type": "text"
+          }
+        }
+      },
+      "path": {
+        "properties": {
+          "real": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          },
+          "root": {
+            "type": "keyword"
+          },
+          "virtual": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

es_kibana/config/_default/7/_settings_folder.json

@@ -0,0 +1,32 @@
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "fscrawler_path": {
+          "tokenizer": "fscrawler_path"
+        }
+      },
+      "tokenizer": {
+        "fscrawler_path": {
+          "type": "path_hierarchy"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "properties" : {
+      "real" : {
+        "type" : "keyword",
+        "store" : true
+      },
+      "root" : {
+        "type" : "keyword",
+        "store" : true
+      },
+      "virtual" : {
+        "type" : "keyword",
+        "store" : true
+      }
+    }
+  }
+}

es_kibana/config/_settings.yaml

@@ -0,0 +1,34 @@
+---
+name: "tuffy_es"
+fs:
+  url: "/tmp/es"
+  update_rate: "5s"
+  includes:
+  - "*.doc"
+  - "*.pdf"
+  json_support: false 
+  filename_as_id: false
+  add_filesize: true
+  remove_deleted: true
+  add_as_inner_object: false
+  store_source: false
+  index_content: true
+  attributes_support: false
+  raw_metadata: true 
+  xml_support: false 
+  index_folders: true
+  lang_detect: true 
+  continue_on_error: true
+  ocr:
+    language: "eng"
+    enabled: true
+    pdf_strategy: "ocr_and_text"
+  follow_symlinks: false
+elasticsearch:
+  nodes:
+    - url: "http://es01:9200"
+    - url: "http://es02:9200"
+    - url: "http://es03:9200"
+  bulk_size: 100
+  flush_interval: "5s"
+  byte_size: "2mb"

es_kibana/config/tuffy_es/_default/6/_settings.json

@@ -0,0 +1,214 @@
+{
+  "settings": {
+    "number_of_shards": 1,
+    "index.mapping.total_fields.limit": 2000,
+    "analysis": {
+      "analyzer": {
+        "fscrawler_path": {
+          "tokenizer": "fscrawler_path"
+        }
+      },
+      "tokenizer": {
+        "fscrawler_path": {
+          "type": "path_hierarchy"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "raw_as_text": {
+          "path_match": "meta.raw.*",
+          "mapping": {
+            "type": "text",
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            }
+          }
+        }
+      }
+    ],
+    "properties": {
+      "attachment": {
+        "type": "binary",
+        "doc_values": false
+      },
+      "attributes": {
+        "properties": {
+          "group": {
+            "type": "keyword"
+          },
+          "owner": {
+            "type": "keyword"
+          }
+        }
+      },
+      "content": {
+        "type": "text"
+      },
+      "file": {
+        "properties": {
+          "content_type": {
+            "type": "keyword"
+          },
+          "filename": {
+            "type": "keyword",
+            "store": true
+          },
+          "extension": {
+            "type": "keyword"
+          },
+          "filesize": {
+            "type": "long"
+          },
+          "indexed_chars": {
+            "type": "long"
+          },
+          "indexing_date": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "created": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "last_modified": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "last_accessed": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "checksum": {
+            "type": "keyword"
+          },
+          "url": {
+            "type": "keyword",
+            "index": false
+          }
+        }
+      },
+      "meta": {
+        "properties": {
+          "author": {
+            "type": "text"
+          },
+          "date": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "keywords": {
+            "type": "text"
+          },
+          "title": {
+            "type": "text"
+          },
+          "language": {
+            "type": "keyword"
+          },
+          "format": {
+            "type": "text"
+          },
+          "identifier": {
+            "type": "text"
+          },
+          "contributor": {
+            "type": "text"
+          },
+          "coverage": {
+            "type": "text"
+          },
+          "modifier": {
+            "type": "text"
+          },
+          "creator_tool": {
+            "type": "keyword"
+          },
+          "publisher": {
+            "type": "text"
+          },
+          "relation": {
+            "type": "text"
+          },
+          "rights": {
+            "type": "text"
+          },
+          "source": {
+            "type": "text"
+          },
+          "type": {
+            "type": "text"
+          },
+          "description": {
+            "type": "text"
+          },
+          "created": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "print_date": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "metadata_date": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "latitude": {
+            "type": "text"
+          },
+          "longitude": {
+            "type": "text"
+          },
+          "altitude": {
+            "type": "text"
+          },
+          "rating": {
+            "type": "byte"
+          },
+          "comments": {
+            "type": "text"
+          }
+        }
+      },
+      "path": {
+        "properties": {
+          "real": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          },
+          "root": {
+            "type": "keyword"
+          },
+          "virtual": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

es_kibana/config/tuffy_es/_default/6/_settings_folder.json

@@ -0,0 +1,32 @@
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "fscrawler_path": {
+          "tokenizer": "fscrawler_path"
+        }
+      },
+      "tokenizer": {
+        "fscrawler_path": {
+          "type": "path_hierarchy"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "properties" : {
+      "real" : {
+        "type" : "keyword",
+        "store" : true
+      },
+      "root" : {
+        "type" : "keyword",
+        "store" : true
+      },
+      "virtual" : {
+        "type" : "keyword",
+        "store" : true
+      }
+    }
+  }
+}

es_kibana/config/tuffy_es/_default/7/_settings.json

@@ -0,0 +1,214 @@
+{
+  "settings": {
+    "number_of_shards": 1,
+    "index.mapping.total_fields.limit": 2000,
+    "analysis": {
+      "analyzer": {
+        "fscrawler_path": {
+          "tokenizer": "fscrawler_path"
+        }
+      },
+      "tokenizer": {
+        "fscrawler_path": {
+          "type": "path_hierarchy"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "dynamic_templates": [
+      {
+        "raw_as_text": {
+          "path_match": "meta.raw.*",
+          "mapping": {
+            "type": "text",
+            "fields": {
+              "keyword": {
+                "type": "keyword",
+                "ignore_above": 256
+              }
+            }
+          }
+        }
+      }
+    ],
+    "properties": {
+      "attachment": {
+        "type": "binary",
+        "doc_values": false
+      },
+      "attributes": {
+        "properties": {
+          "group": {
+            "type": "keyword"
+          },
+          "owner": {
+            "type": "keyword"
+          }
+        }
+      },
+      "content": {
+        "type": "text"
+      },
+      "file": {
+        "properties": {
+          "content_type": {
+            "type": "keyword"
+          },
+          "filename": {
+            "type": "keyword",
+            "store": true
+          },
+          "extension": {
+            "type": "keyword"
+          },
+          "filesize": {
+            "type": "long"
+          },
+          "indexed_chars": {
+            "type": "long"
+          },
+          "indexing_date": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "created": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "last_modified": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "last_accessed": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "checksum": {
+            "type": "keyword"
+          },
+          "url": {
+            "type": "keyword",
+            "index": false
+          }
+        }
+      },
+      "meta": {
+        "properties": {
+          "author": {
+            "type": "text"
+          },
+          "date": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "keywords": {
+            "type": "text"
+          },
+          "title": {
+            "type": "text"
+          },
+          "language": {
+            "type": "keyword"
+          },
+          "format": {
+            "type": "text"
+          },
+          "identifier": {
+            "type": "text"
+          },
+          "contributor": {
+            "type": "text"
+          },
+          "coverage": {
+            "type": "text"
+          },
+          "modifier": {
+            "type": "text"
+          },
+          "creator_tool": {
+            "type": "keyword"
+          },
+          "publisher": {
+            "type": "text"
+          },
+          "relation": {
+            "type": "text"
+          },
+          "rights": {
+            "type": "text"
+          },
+          "source": {
+            "type": "text"
+          },
+          "type": {
+            "type": "text"
+          },
+          "description": {
+            "type": "text"
+          },
+          "created": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "print_date": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "metadata_date": {
+            "type": "date",
+            "format": "dateOptionalTime"
+          },
+          "latitude": {
+            "type": "text"
+          },
+          "longitude": {
+            "type": "text"
+          },
+          "altitude": {
+            "type": "text"
+          },
+          "rating": {
+            "type": "byte"
+          },
+          "comments": {
+            "type": "text"
+          }
+        }
+      },
+      "path": {
+        "properties": {
+          "real": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          },
+          "root": {
+            "type": "keyword"
+          },
+          "virtual": {
+            "type": "keyword",
+            "fields": {
+              "tree": {
+                "type": "text",
+                "analyzer": "fscrawler_path",
+                "fielddata": true
+              },
+              "fulltext": {
+                "type": "text"
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+}

es_kibana/config/tuffy_es/_default/7/_settings_folder.json

@@ -0,0 +1,32 @@
+{
+  "settings": {
+    "analysis": {
+      "analyzer": {
+        "fscrawler_path": {
+          "tokenizer": "fscrawler_path"
+        }
+      },
+      "tokenizer": {
+        "fscrawler_path": {
+          "type": "path_hierarchy"
+        }
+      }
+    }
+  },
+  "mappings": {
+    "properties" : {
+      "real" : {
+        "type" : "keyword",
+        "store" : true
+      },
+      "root" : {
+        "type" : "keyword",
+        "store" : true
+      },
+      "virtual" : {
+        "type" : "keyword",
+        "store" : true
+      }
+    }
+  }
+}

es_kibana/config/tuffy_es/_settings.yaml

@@ -0,0 +1,37 @@
+---
+name: "tuffy_es"
+fs:
+  url: "/tmp/es"
+  update_rate: "5s"
+  includes:
+  - "*/*"
+  - "*.doc"
+  - "*.pdf"
+  json_support: false 
+  filename_as_id: false
+  add_filesize: true
+  remove_deleted: true
+  add_as_inner_object: false
+  store_source: false
+  index_content: true
+  attributes_support: false
+  raw_metadata: true 
+  xml_support: false 
+  index_folders: true
+  lang_detect: true 
+  continue_on_error: true
+  ocr:
+    language: "eng"
+    enabled: true
+    pdf_strategy: "ocr_and_text"
+  follow_symlinks: false
+elasticsearch:
+  nodes:
+    - url: "http://es01:9200"
+    - url: "http://es02:9200"
+    - url: "http://es03:9200"
+  bulk_size: 100
+  flush_interval: "5s"
+  byte_size: "2mb"
+rest:
+    url: "http://fscrawler:8080"

es_kibana/data/files01/nodes

@@ -0,0 +1 @@
+written by Elasticsearch v8.4.3 to prevent a downgrade to a version prior to v8.0.0 which would result in data loss