Added new Setting property UnmodifiableOnRestore to prevent updating settings on restore snapshot

CHANGELOG.md

@@ -33,6 +33,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Support searching from doc_value using termQueryCaseInsensitive/termQuery in flat_object/keyword field([#16974](https://github.com/opensearch-project/OpenSearch/pull/16974/))
 - Added a new `time` field to replace the deprecated `getTime` field in `GetStats`. ([#17009](https://github.com/opensearch-project/OpenSearch/pull/17009))
 - Improve performance of the bitmap filtering([#16936](https://github.com/opensearch-project/OpenSearch/pull/16936/))
+- Added new Setting property UnmodifiableOnRestore to prevent updating settings on restore snapshot ([#16957](https://github.com/opensearch-project/OpenSearch/pull/16957))
 
 ### Dependencies
 - Bump `com.google.cloud:google-cloud-core-http` from 2.23.0 to 2.47.0 ([#16504](https://github.com/opensearch-project/OpenSearch/pull/16504))

server/src/main/java/org/opensearch/cluster/metadata/IndexMetadata.java

@@ -230,7 +230,15 @@ static Setting<Integer> buildNumberOfShardsSetting() {
                     + "]"
             );
         }
-        return Setting.intSetting(SETTING_NUMBER_OF_SHARDS, defaultNumShards, 1, maxNumShards, Property.IndexScope, Property.Final);
+        return Setting.intSetting(
+            SETTING_NUMBER_OF_SHARDS,
+            defaultNumShards,
+            1,
+            maxNumShards,
+            Property.IndexScope,
+            Property.Final,
+            Property.UnmodifiableOnRestore
+        );
     }
 
     public static final String INDEX_SETTING_PREFIX = "index.";
@@ -559,13 +567,15 @@ public static APIBlock readFrom(StreamInput input) throws IOException {
         SETTING_VERSION_CREATED,
         Version.V_EMPTY,
         Property.IndexScope,
-        Property.PrivateIndex
+        Property.PrivateIndex,
+        Property.UnmodifiableOnRestore
     );
 
     public static final String SETTING_VERSION_CREATED_STRING = "index.version.created_string";
     public static final String SETTING_VERSION_UPGRADED = "index.version.upgraded";
     public static final String SETTING_VERSION_UPGRADED_STRING = "index.version.upgraded_string";
     public static final String SETTING_CREATION_DATE = "index.creation_date";
+
     /**
      * The user provided name for an index. This is the plain string provided by the user when the index was created.
      * It might still contain date math expressions etc. (added in 5.0)
@@ -589,6 +599,7 @@ public static APIBlock readFrom(StreamInput input) throws IOException {
         Function.identity(),
         Property.IndexScope
     );
+
     public static final String INDEX_UUID_NA_VALUE = Strings.UNKNOWN_UUID_VALUE;
 
     public static final String INDEX_ROUTING_REQUIRE_GROUP_PREFIX = "index.routing.allocation.require";

server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java

@@ -231,6 +231,10 @@ public MetadataCreateIndexService(
             : null;
     }
 
+    public IndexScopedSettings getIndexScopedSettings() {
+        return indexScopedSettings;
+    }
+
     /**
      * Add a provider to be invoked to get additional index settings prior to an index being created
      */

server/src/main/java/org/opensearch/common/settings/AbstractScopedSettings.java

@@ -759,6 +759,14 @@ public boolean isFinalSetting(String key) {
         return setting != null && setting.isFinal();
     }
 
+    /**
+     * Returns <code>true</code> if the setting for the given key is unmodifiableOnRestore. Otherwise <code>false</code>.
+     */
+    public boolean isUnmodifiableOnRestoreSetting(String key) {
+        final Setting<?> setting = get(key);
+        return setting != null && setting.isUnmodifiableOnRestore();
+    }
+
     /**
      * Returns a settings object that contains all settings that are not
      * already set in the given source. The diff contains either the default value for each

server/src/main/java/org/opensearch/common/settings/Setting.java

@@ -171,7 +171,12 @@ public enum Property {
         /**
          * Extension scope
          */
-        ExtensionScope
+        ExtensionScope,
+
+        /**
+         * Mark this setting as immutable on snapshot restore
+         */
+        UnmodifiableOnRestore
     }
 
     private final Key key;
@@ -208,7 +213,16 @@ private Setting(
             final EnumSet<Property> propertiesAsSet = EnumSet.copyOf(Arrays.asList(properties));
             if (propertiesAsSet.contains(Property.Dynamic) && propertiesAsSet.contains(Property.Final)) {
                 throw new IllegalArgumentException("final setting [" + key + "] cannot be dynamic");
-            }
+            } else if (propertiesAsSet.contains(Property.UnmodifiableOnRestore)
+                && (propertiesAsSet.contains(Property.Dynamic) || !propertiesAsSet.contains(Property.IndexScope))) {
+                    throw new IllegalArgumentException(
+                        "unmodifiableOnRestore setting ["
+                            + key
+                            + "] cannot be dynamic, or unmodifiableOnRestore setting ["
+                            + key
+                            + "] must have indexScope"
+                    );
+                }
             checkPropertyRequiresIndexScope(propertiesAsSet, Property.NotCopyableOnResize);
             checkPropertyRequiresIndexScope(propertiesAsSet, Property.InternalIndex);
             checkPropertyRequiresIndexScope(propertiesAsSet, Property.PrivateIndex);
@@ -348,6 +362,10 @@ public final boolean isFinal() {
         return properties.contains(Property.Final);
     }
 
+    public final boolean isUnmodifiableOnRestore() {
+        return properties.contains(Property.UnmodifiableOnRestore);
+    }
+
     public final boolean isInternalIndex() {
         return properties.contains(Property.InternalIndex);
     }

server/src/main/java/org/opensearch/snapshots/RestoreService.java

@@ -77,6 +77,7 @@
 import org.opensearch.common.lucene.Lucene;
 import org.opensearch.common.regex.Regex;
 import org.opensearch.common.settings.ClusterSettings;
+import org.opensearch.common.settings.IndexScopedSettings;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.unit.TimeValue;
 import org.opensearch.common.util.ArrayUtils;
@@ -122,12 +123,10 @@
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_INDEX_UUID;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SEARCH_REPLICAS;
-import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_SEGMENT_STORE_REPOSITORY;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_STORE_ENABLED;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REMOTE_TRANSLOG_STORE_REPOSITORY;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_REPLICATION_TYPE;
-import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_CREATED;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_VERSION_UPGRADED;
 import static org.opensearch.common.util.FeatureFlags.SEARCHABLE_SNAPSHOT_EXTENDED_COMPATIBILITY;
 import static org.opensearch.common.util.IndexUtils.filterIndices;
@@ -164,8 +163,6 @@
 
     private static final Set<String> USER_UNMODIFIABLE_SETTINGS = unmodifiableSet(
         newHashSet(
-            SETTING_NUMBER_OF_SHARDS,
-            SETTING_VERSION_CREATED,
             SETTING_INDEX_UUID,
             SETTING_CREATION_DATE,
             SETTING_HISTORY_UUID,
@@ -180,7 +177,7 @@
     private static final String REMOTE_STORE_INDEX_SETTINGS_REGEX = "index.remote_store.*";
 
     static {
-        Set<String> unremovable = new HashSet<>(USER_UNMODIFIABLE_SETTINGS.size() + 4);
+        Set<String> unremovable = new HashSet<>(USER_UNMODIFIABLE_SETTINGS.size() + 3);
         unremovable.addAll(USER_UNMODIFIABLE_SETTINGS);
         unremovable.add(SETTING_NUMBER_OF_REPLICAS);
         unremovable.add(SETTING_AUTO_EXPAND_REPLICAS);
@@ -202,6 +199,8 @@
 
     private final ClusterSettings clusterSettings;
 
+    private final IndexScopedSettings indexScopedSettings;
+
     private final IndicesService indicesService;
 
     private final Supplier<ClusterInfo> clusterInfoSupplier;
@@ -234,6 +233,7 @@
         this.clusterSettings = clusterService.getClusterSettings();
         this.shardLimitValidator = shardLimitValidator;
         this.indicesService = indicesService;
+        this.indexScopedSettings = createIndexService.getIndexScopedSettings();
         this.clusterInfoSupplier = clusterInfoSupplier;
         this.dataToFileCacheSizeRatioSupplier = dataToFileCacheSizeRatioSupplier;
 
@@ -835,6 +835,11 @@
                                         snapshot,
                                         "cannot remove setting [" + ignoredSetting + "] on restore"
                                     );
+                                } else if (indexScopedSettings.isUnmodifiableOnRestoreSetting(ignoredSetting)) {
+                                    throw new SnapshotRestoreException(
+                                        snapshot,
+                                        "cannot remove UnmodifiableOnRestore setting [" + ignoredSetting + "] on restore"
+                                    );
                                 } else {
                                     keyFilters.add(ignoredSetting);
                                 }
@@ -853,7 +858,7 @@
                         }
 
                         Predicate<String> settingsFilter = k -> {
-                            if (USER_UNREMOVABLE_SETTINGS.contains(k) == false) {
+                            if (USER_UNREMOVABLE_SETTINGS.contains(k) == false && !indexScopedSettings.isUnmodifiableOnRestoreSetting(k)) {
                                 for (String filterKey : keyFilters) {
                                     if (k.equals(filterKey)) {
                                         return false;
@@ -872,6 +877,11 @@
                             .put(normalizedChangeSettings.filter(k -> {
                                 if (USER_UNMODIFIABLE_SETTINGS.contains(k)) {
                                     throw new SnapshotRestoreException(snapshot, "cannot modify setting [" + k + "] on restore");
+                                } else if (indexScopedSettings.isUnmodifiableOnRestoreSetting(k)) {
+                                    throw new SnapshotRestoreException(
+                                        snapshot,
+                                        "cannot modify UnmodifiableOnRestore setting [" + k + "] on restore"
+                                    );
                                 } else {
                                     return true;
                                 }

server/src/test/java/org/opensearch/common/settings/ScopedSettingsTests.java

@@ -789,6 +789,30 @@ public void testIsFinal() {
         assertTrue(settings.isFinalSetting("foo.group.key"));
     }
 
+    public void testIsUnmodifiableOnRestore() {
+        ClusterSettings settings = new ClusterSettings(
+            Settings.EMPTY,
+            new HashSet<>(
+                Arrays.asList(
+                    Setting.intSetting("foo.int", 1, Property.UnmodifiableOnRestore, Property.IndexScope, Property.NodeScope),
+                    Setting.groupSetting("foo.group.", Property.UnmodifiableOnRestore, Property.IndexScope, Property.NodeScope),
+                    Setting.groupSetting("foo.list.", Property.UnmodifiableOnRestore, Property.IndexScope, Property.NodeScope),
+                    Setting.intSetting("foo.int.baz", 1, Property.IndexScope, Property.NodeScope)
+                )
+            )
+        );
+
+        assertFalse(settings.isUnmodifiableOnRestoreSetting("foo.int.baz"));
+        assertTrue(settings.isUnmodifiableOnRestoreSetting("foo.int"));
+
+        assertFalse(settings.isUnmodifiableOnRestoreSetting("foo.list"));
+        assertTrue(settings.isUnmodifiableOnRestoreSetting("foo.list.0.key"));
+        assertTrue(settings.isUnmodifiableOnRestoreSetting("foo.list.key"));
+
+        assertFalse(settings.isUnmodifiableOnRestoreSetting("foo.group"));
+        assertTrue(settings.isUnmodifiableOnRestoreSetting("foo.group.key"));
+    }
+
     public void testDiff() throws IOException {
         Setting<Integer> fooBarBaz = Setting.intSetting("foo.bar.baz", 1, Property.NodeScope);
         Setting<Integer> fooBar = Setting.intSetting("foo.bar", 1, Property.Dynamic, Property.NodeScope);

server/src/test/java/org/opensearch/common/settings/SettingTests.java

@@ -1439,6 +1439,32 @@ public void testRejectConflictingDynamicAndFinalProperties() {
         assertThat(ex.getMessage(), containsString("final setting [foo.bar] cannot be dynamic"));
     }
 
+    public void testRejectConflictingDynamicAndUnmodifiableOnRestoreProperties() {
+        IllegalArgumentException ex = expectThrows(
+            IllegalArgumentException.class,
+            () -> Setting.simpleString("foo.bar", Property.UnmodifiableOnRestore, Property.Dynamic)
+        );
+        assertThat(
+            ex.getMessage(),
+            containsString(
+                "unmodifiableOnRestore setting [foo.bar] cannot be dynamic, or unmodifiableOnRestore setting [foo.bar] must have indexScope"
+            )
+        );
+    }
+
+    public void testRejectMissingIndexScopeAndUnmodifiableOnRestoreProperties() {
+        IllegalArgumentException ex = expectThrows(
+            IllegalArgumentException.class,
+            () -> Setting.simpleString("foo.bar", Property.UnmodifiableOnRestore)
+        );
+        assertThat(
+            ex.getMessage(),
+            containsString(
+                "unmodifiableOnRestore setting [foo.bar] cannot be dynamic, or unmodifiableOnRestore setting [foo.bar] must have indexScope"
+            )
+        );
+    }
+
     public void testRejectNonIndexScopedNotCopyableOnResizeSetting() {
         final IllegalArgumentException e = expectThrows(
             IllegalArgumentException.class,