Skip to content

Commit 9e081a9

Browse files
enoch85enoch85
authored
Change to ElasticSearch for FTS (#2541)
1 parent b50a04c commit 9e081a9

File tree

3 files changed

+61
-176
lines changed

3 files changed

+61
-176
lines changed

‎apps/fulltextsearch.sh‎

Lines changed: 46 additions & 155 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,13 @@
55

66
true
77
SCRIPT_NAME="Full Text Search"
8-
SCRIPT_EXPLAINER="Full Text Search provides OpenSearch for Nextcloud, which makes it possible to search for text inside files."
8+
SCRIPT_EXPLAINER="Full Text Search provides ElastichSearch for Nextcloud, which makes it possible to search for text inside files."
99
# shellcheck source=lib.sh
1010
source /var/scripts/fetch_lib.sh
1111

1212
# Get all needed variables from the library
13-
ncdb
1413
nc_update
15-
opensearch_install
16-
ncdomain
14+
fulltextsearch_install
1715

1816
# Check for errors + debug code and abort if something isn't right
1917
# 1 = ON
@@ -28,14 +26,9 @@ root_check
2826
lowest_compatible_nc 21
2927

3028
# Check if Full Text Search is already installed
31-
if ! does_this_docker_exist "$nc_fts" && ! does_this_docker_exist "$opens_fts" && ! is_app_installed fulltextsearch
29+
if ! does_this_docker_exist docker.elastic.co/elasticsearch/elasticsearch && ! is_app_installed fulltextsearch
3230
then
3331
# Ask for installing
34-
if [ "${CURRENTVERSION%%.*}" -ge "25" ]
35-
then
36-
msg_box "Sorry, it's not possible to install FTS anymore since Nextcloud decided to remove support for OpenSearch. Read more in this issue: https://github.com/nextcloud/fulltextsearch_elasticsearch/issues/271"
37-
exit 1
38-
fi
3932
install_popup "$SCRIPT_NAME"
4033
else
4134
# Ask for removal or reinstallation
@@ -56,16 +49,11 @@ else
5649
nextcloud_occ app:remove "$app"
5750
fi
5851
done
59-
# Removal Docker image
60-
docker_prune_this "$nc_fts"
61-
docker_prune_volume "esdata"
62-
docker-compose_down "$OPNSDIR/docker-compose.yml"
63-
# Remove configuration files
64-
rm -rf "$RORDIR"
65-
rm -rf "$OPNSDIR"
52+
# Removal Elastichsearch Docker image
53+
docker_prune_this "docker.elastic.co/elasticsearch/elasticsearch"
54+
rm -rf "$FULLTEXTSEARCH_DIR"
6655
# Show successful uninstall if applicable
6756
removal_popup "$SCRIPT_NAME"
68-
apt-get purge docker-compose -y
6957
fi
7058

7159
# Test RAM size (4GB min) + CPUs (min 2)
@@ -95,6 +83,18 @@ then
9583
deluser --group solr
9684
fi
9785

86+
# Removal Opensearch Docker image
87+
if does_this_docker_exist "$nc_fts" && does_this_docker_exist "$opens_fts"
88+
then
89+
docker_prune_this "$nc_fts"
90+
docker_prune_volume "esdata"
91+
docker-compose_down "$OPNSDIR/docker-compose.yml"
92+
# Remove configuration files
93+
rm -rf "$RORDIR"
94+
rm -rf "$OPNSDIR"
95+
apt-get purge docker-compose -y
96+
fi
97+
9898
# Check if the app is compatible with the current Nextcloud version
9999
if ! install_and_enable_app fulltextsearch
100100
then
@@ -103,173 +103,64 @@ fi
103103

104104
# Check & install docker
105105
install_docker
106-
install_if_not docker-compose
107106
set_max_count
108-
mkdir -p "$OPNSDIR"
109-
docker pull "$opens_fts"
110-
BCRYPT_HASH="$(docker run --rm -it $opens_fts \
111-
bash -c "plugins/opensearch-security/tools/hash.sh -p $OPNSREST | tr -d ':\n' ")"
112-
113-
# Create configurations YML
114-
# opensearch.yml
115-
cat << YML_OPENSEARCH > $OPNSDIR/opensearch.yml
116-
cluster.name: docker-cluster
117-
# Avoid Docker assigning IP.
118-
network.host: 0.0.0.0
119107

120-
# Declaring single node cluster.
121-
discovery.type: single-node
122-
123-
######## Start Security Configuration ########
124-
plugins.security.ssl.transport.pemcert_filepath: node.pem
125-
plugins.security.ssl.transport.pemkey_filepath: node-key.pem
126-
plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem
127-
plugins.security.ssl.transport.enforce_hostname_verification: false
128-
129-
# Disable ssl at REST as Fulltextsearch can't accept self-signed CA certs.
130-
plugins.security.ssl.http.enabled: false
131-
#plugins.security.ssl.http.pemcert_filepath: node.pem
132-
#plugins.security.ssl.http.pemkey_filepath: node-key.pem
133-
#plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem
134-
plugins.security.allow_unsafe_democertificates: false
135-
plugins.security.allow_default_init_securityindex: true
136-
plugins.security.authcz.admin_dn:
137-
- 'CN=admin,OU=FTS,O=OPENSEARCH,L=VM,ST=NEXTCLOUD,C=CA'
138-
plugins.security.nodes_dn:
139-
- 'CN=${NCDOMAIN},OU=FTS,O=OPENSEARCH,L=VM,ST=NEXTCLOUD,C=CA'
140-
141-
plugins.security.audit.type: internal_opensearch
142-
plugins.security.enable_snapshot_restore_privilege: true
143-
plugins.security.check_snapshot_restore_write_privileges: true
144-
plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]
145-
plugins.security.system_indices.enabled: true
146-
plugins.security.system_indices.indices: [".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opendistro-notifications-*", ".opendistro-notebooks", ".opensearch-observability", ".opendistro-asynchronous-search-response*", ".replication-metadata-store"]
147-
node.max_local_storage_nodes: 1
148-
######## End Security Configuration ########
149-
YML_OPENSEARCH
150-
151-
# internal_users.yml
152-
cat << YML_INTERNAL_USERS > $OPNSDIR/internal_users.yml
153-
_meta:
154-
type: "internalusers"
155-
config_version: 2
156-
157-
${INDEX_USER}:
158-
hash: "${BCRYPT_HASH}"
159-
reserved: true
160-
backend_roles:
161-
- "admin"
162-
description: "admin user for fts at opensearch."
163-
YML_INTERNAL_USERS
164-
165-
# roles_mapping.yml
166-
cat << YML_ROLES_MAPPING > $OPNSDIR/roles_mapping.yml
167-
_meta:
168-
type: "rolesmapping"
169-
config_version: 2
170-
171-
# Roles mapping
172-
all_access:
173-
reserved: false
174-
backend_roles:
175-
- "admin"
176-
description: "Maps admin to all_access"
177-
YML_ROLES_MAPPING
178-
179-
# docker-compose.yml
180-
cat << YML_DOCKER_COMPOSE > $OPNSDIR/docker-compose.yml
108+
mkdir -p "$FULLTEXTSEARCH_DIR"
109+
cat << YML_DOCKER_COMPOSE > "$FULLTEXTSEARCH_DIR/docker-compose.yaml"
181110
version: '3'
182111
services:
183-
fts_os-node:
184-
image: opensearchproject/opensearch:1
185-
container_name: fts_os-node
112+
elasticsearch:
113+
image: docker.elastic.co/elasticsearch/elasticsearch:8.8.1
114+
container_name: $DOCKER_IMAGE_NAME
186115
restart: always
187-
command:
188-
- sh
189-
- -c
190-
- "/usr/share/opensearch/bin/opensearch-plugin list | grep -q ingest-attachment \
191-
|| /usr/share/opensearch/bin/opensearch-plugin install --batch ingest-attachment ;
192-
./opensearch-docker-entrypoint.sh"
116+
ports:
117+
- 127.0.0.1:9200:9200
193118
environment:
194-
- cluster.name=fts_os-cluster
195-
- node.name=fts_os-node
196-
- bootstrap.memory_lock=true
197-
- "OPENSEARCH_JAVA_OPTS=-Xms1024M -Xmx1024M"
119+
- discovery.type=single-node
120+
- xpack.security.enabled=true
121+
- xpack.security.http.ssl.enabled=false
122+
- ELASTIC_PASSWORD=$ELASTIC_USER_PASSWORD
198123
ulimits:
199124
memlock:
200125
soft: -1
201126
hard: -1
202127
nofile:
203128
soft: 65536
204129
hard: 65536
205-
volumes:
206-
- fts_os-data:/usr/share/opensearch/data
207-
- $OPNSDIR/root-ca.pem:/usr/share/opensearch/config/root-ca.pem
208-
- $OPNSDIR/node.pem:/usr/share/opensearch/config/node.pem
209-
- $OPNSDIR/node-key.pem:/usr/share/opensearch/config/node-key.pem
210-
- $OPNSDIR/admin.pem:/usr/share/opensearch/config/admin.pem
211-
- $OPNSDIR/admin-key.pem:/usr/share/opensearch/config/admin-key.pem
212-
- $OPNSDIR/opensearch.yml:/usr/share/opensearch/config/opensearch.yml
213-
- $OPNSDIR/internal_users.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/internal_users.yml
214-
- $OPNSDIR/roles_mapping.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/roles_mapping.yml
215-
ports:
216-
- 127.0.0.1:9200:9200
217-
- 127.0.0.1:9600:9600 # Performance Analyzer [1]
218130
networks:
219-
- fts_os-net
131+
- $DOCKER_IMAGE_NAME-network
220132
221133
volumes:
222-
fts_os-data:
223-
134+
$DOCKER_IMAGE_NAME-data:
224135
networks:
225-
fts_os-net:
226-
227-
#[1] https://github.com/opensearch-project/performance-analyzer
136+
$DOCKER_IMAGE_NAME-network:
228137
YML_DOCKER_COMPOSE
229138

230-
# Prepare certs
231-
create_certs "$NCDOMAIN"
232-
233-
# Set permissions
234-
chmod 744 -R $OPNSDIR
139+
# Start the docker image
140+
cd "$FULLTEXTSEARCH_DIR"
141+
docker compose up -d
235142

236-
# Launch docker-compose
237-
cd $OPNSDIR
238-
docker-compose up -d
239-
240-
# Wait for bootstrapping
241-
if [ "$(nproc)" -gt 2 ]
242-
then
243-
countdown "Waiting for Docker bootstrapping..." "60"
244-
else
245-
countdown "Waiting for Docker bootstrapping..." "120"
246-
fi
143+
# Check if online
144+
until curl -sS "http://elastic:$ELASTIC_USER_PASSWORD@localhost:9200/_cat/health?h=status" | grep -q "green\|yellow"
145+
do
146+
countdown "Waiting for ElasticSearch to come online..." "3"
147+
done
247148

248-
# Make sure password setup is enforced.
249-
docker-compose exec fts_os-node \
250-
bash -c "cd \
251-
plugins/opensearch-security/tools/ && \
252-
bash securityadmin.sh -f \
253-
../securityconfig/internal_users.yml \
254-
-t internalusers \
255-
-icl \
256-
-nhnv \
257-
-cacert ../../../config/root-ca.pem \
258-
-cert ../../../config/admin.pem \
259-
-key ../../../config/admin-key.pem && \
260-
chmod 0600 ../../../config/root-ca.pem ../../../config/admin.pem ../../../config/admin-key.pem"
149+
# Check logs
150+
print_text_in_color "$ICyan" "Checking logs..."
151+
docker logs "$DOCKER_IMAGE_NAME"
261152

262-
docker logs $fts_node
153+
countdown "Waiting a bit more before testing..." "10"
263154

264155
# Get Full Text Search app for nextcloud
265156
install_and_enable_app fulltextsearch
266157
install_and_enable_app fulltextsearch_elasticsearch
267158
install_and_enable_app files_fulltextsearch
268-
chown -R www-data:www-data $NC_APPS_PATH
159+
chown -R www-data:www-data "$NC_APPS_PATH"
269160

270161
# Final setup
271162
nextcloud_occ fulltextsearch:configure '{"search_platform":"OCA\\FullTextSearch_Elasticsearch\\Platform\\ElasticSearchPlatform"}'
272-
nextcloud_occ fulltextsearch_elasticsearch:configure "{\"elastic_host\":\"http://${INDEX_USER}:${OPNSREST}@localhost:9200\",\"elastic_index\":\"${INDEX_USER}-index\"}"
163+
nextcloud_occ fulltextsearch_elasticsearch:configure "{\"elastic_host\":\"http://elastic:$ELASTIC_USER_PASSWORD@localhost:9200\",\"elastic_index\":\"${NEXTCLOUD_INDEX}\"}"
273164
nextcloud_occ files_fulltextsearch:configure "{\"files_pdf\":\"1\",\"files_office\":\"1\"}"
274165

275166
# Wait further for cache for index to work

‎lib.sh‎

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -173,20 +173,18 @@ SPAMHAUS=/etc/spamhaus.wl
173173
ENVASIVE=/etc/apache2/mods-available/mod-evasive.load
174174
APACHE2=/etc/apache2/apache2.conf
175175
# Full text Search
176-
opensearch_install() {
177-
INDEX_USER=$(gen_passwd "$SHUF" '[:lower:]')
178-
OPNSREST=$(gen_passwd "$SHUF" "A-Za-z0-9")
176+
fulltextsearch_install() {
177+
NEXTCLOUD_INDEX=$(gen_passwd "$SHUF" '[:lower:]')
178+
ELASTIC_USER_PASSWORD=$(gen_passwd "$SHUF" '[:lower:]')
179+
DOCKER_IMAGE_NAME=es01
180+
FULLTEXTSEARCH_DIR="$SCRIPTS"/fulltextsearch
181+
# Legacy
179182
nc_fts="ark74/nc_fts"
183+
nc_fts="ark74/nc_fts"
184+
opens_fts="opensearchproject/opensearch:1"
180185
opens_fts="opensearchproject/opensearch:1"
181186
fts_node="fts_os-node"
182-
}
183-
create_certs(){
184-
download_script APP opensearch_certs
185-
check_command sed -i "s|__NCDOMAIN__|$1|" "$SCRIPTS"/opensearch_certs.sh
186-
check_command mv "$SCRIPTS"/opensearch_certs.sh "$OPNSDIR"
187-
check_command cd "$OPNSDIR"
188-
check_command bash opensearch_certs.sh
189-
rm -f "$OPNSDIR"/opensearch_certs.sh
187+
fts_node="fts_os-node"
190188
}
191189
# Name in trusted_config
192190
ncdomain() {
@@ -1798,14 +1796,14 @@ then
17981796
echo "Docker image just got updated! We just updated $2 docker image automatically! $(date +%Y%m%d)" >> "$VMLOGS"/update.log
17991797
fi
18001798
}
1801-
# docker-compose_update 'fts_os-node' 'Full Text Search' "$OPNSDIR"
1799+
# docker-compose_update 'fulltextsearch-elasticsearch' 'Full Text Search' "$FTSDIR"
18021800
# (docker conainter name = $1, the name in text = $2 , docker-compose directory = $3)
18031801
docker-compose_update() {
18041802
if is_docker_running && docker ps -a --format "{{.Names}}" | grep -q "^$1$"
18051803
then
18061804
cd "$3"
1807-
docker-compose pull
1808-
docker-compose up -d --remove-orphans
1805+
docker compose pull
1806+
docker compose up -d --remove-orphans
18091807
docker image prune -a -f
18101808
print_text_in_color "$IGreen" "$2 docker image just got updated!"
18111809
echo "Docker image just got updated! We just updated $2 docker image automatically! $(date +%Y%m%d)" >> "$VMLOGS"/update.log

‎nextcloud_update.sh‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -651,16 +651,12 @@ If you need help, please get support here: https://shop.hanssonit.se/product/pre
651651
# Full Text Search
652652
if [ "${CURRENTVERSION%%.*}" -ge "25" ]
653653
then
654-
opensearch_install
654+
fulltextsearch_install
655655
if does_this_docker_exist "$nc_fts" && does_this_docker_exist "$opens_fts"
656656
then
657-
msg_box "Sorry, it's not possible to install or upgrade FTS anymore since Nextcloud decided to remove support for OpenSearchh
658-
Read more in this issue: https://github.com/nextcloud/fulltextsearch_elasticsearch/issues/271
659-
660-
Please consider removing it by issuing the uninstall script: sudo bash $SCRIPTS/menu.sh --> Additional Apps --> FullTextSearch"
657+
msg_box "Please consider reinstalling FUllTextSearch since you seem to have the old (and not working) implemantation by issuing the uninstall script: sudo bash $SCRIPTS/menu.sh --> Additional Apps --> FullTextSearch"
661658
else
662-
docker_update_specific 'fts_esror' 'Full Text Search'
663-
docker-compose_update 'fts_os-node' 'Full Text Search' "$OPNSDIR"
659+
docker-compose_update "$DOCKER_IMAGE_NAME" 'Full Text Search' "$FULLTEXTSEARCH_DIR"
664660
fi
665661
fi
666662
# Plex

0 commit comments

Comments
 (0)