fix: Massive insertion on cluster config issues

Fixed Issue #4219

Author: lvca

engine/src/main/java/com/arcadedb/engine/LocalBucket.java

@@ -164,6 +164,14 @@ public LocalBucket(final DatabaseInternal database, final String name, final Str
 
   /**
    * Called at load time.
+   * <p>
+   * Free-space statistics are NOT pre-warmed here. {@link #findAvailableSpace} already calls
+   * {@link #gatherPageStatistics()} lazily on the first allocation that needs it, so a leader
+   * still gets reuse for free; a follower that only applies leader-shipped pages via the state
+   * machine never triggers it at all. Pre-warming here would scan up to all pages of every
+   * bucket during {@link com.arcadedb.schema.LocalSchema#load} - which on a follower under
+   * heavy bulk-load fires repeatedly per LSM compaction SCHEMA_ENTRY and exhausts the heap
+   * (issue #4219).
    */
   public LocalBucket(final DatabaseInternal database, final String name, final String filePath, final int id,
                      final ComponentFile.MODE mode, final int pageSize, final int version) throws IOException {
@@ -178,8 +186,6 @@ public LocalBucket(final DatabaseInternal database, final String name, final Str
     // by default and bypassed the user-DML guard. Schema JSON still maps primary->external by name (which is
     // an orthogonal concern), but the write guard now no longer depends on schema-load ordering.
     this.purpose = purposeForVersion(version);
-    if (this.reuseSpaceMode.ordinal() >= REUSE_SPACE_MODE.HIGH.ordinal())
-      gatherPageStatistics();
   }
 
   private static Purpose purposeForVersion(final int version) {

engine/src/test/java/com/arcadedb/engine/SchemaReloadDoesNotEagerScanBucketsTest.java

@@ -0,0 +1,155 @@
+/*
+ * Copyright © 2021-present Arcade Data Ltd (info@arcadedata.com)
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ * SPDX-FileCopyrightText: 2021-present Arcade Data Ltd (info@arcadedata.com)
+ * SPDX-License-Identifier: Apache-2.0
+ */
+package com.arcadedb.engine;
+
+import com.arcadedb.GlobalConfiguration;
+import com.arcadedb.database.Database;
+import com.arcadedb.database.DatabaseFactory;
+import com.arcadedb.database.DatabaseInternal;
+import com.arcadedb.schema.LocalSchema;
+import com.arcadedb.utility.FileUtils;
+import org.junit.jupiter.api.Test;
+
+import java.io.File;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Regression test for issue #4219.
+ * <p>
+ * On a Raft follower under heavy bulk insertion, every LSM compaction on the leader fires a
+ * SCHEMA_ENTRY that the follower applies via {@code LocalSchema.load(MODE.READ_WRITE, true)}.
+ * Before the fix, that load() path destroyed every in-memory bucket/index instance and the
+ * load-time {@link LocalBucket} constructor called {@code gatherPageStatistics()} eagerly.
+ * For buckets whose pages were mostly full (typical under bulk insert) the scan walked every
+ * single page of every bucket on every reload, pulling them all through the page cache and
+ * the active TransactionContext. Across thousands of compaction events, this exhausted the
+ * follower heap and surfaced as a Java heap space OOM in
+ * {@code com.arcadedb.engine.PageManager.loadPage}, followed by cascading "Bucket with id X
+ * was not found" SchemaExceptions on subsequent TX_ENTRY applies (the schema reload had
+ * already cleared bucketMap before the OOM aborted it).
+ * <p>
+ * After the fix, the load-time bucket constructor no longer pre-warms statistics.
+ * {@link LocalBucket#findAvailableSpace} already calls {@code gatherPageStatistics()} lazily
+ * on the first allocation, so a leader still gets reuse behavior for free; a follower that
+ * only applies leader-shipped page bytes via the state machine never triggers the scan at all.
+ *
+ * @author Luca Garulli (l.garulli@arcadedata.com)
+ */
+class SchemaReloadDoesNotEagerScanBucketsTest {
+
+  private static final String DB_PATH    = "./target/databases/SchemaReloadDoesNotEagerScanBuckets";
+  private static final String TYPE       = "BulkRow";
+  private static final int    REC_COUNT  = 5_000;
+  private static final int    PAYLOAD_KB = 1;
+
+  @Test
+  void schemaReloadDoesNotEagerlyScanBucketPages() throws Exception {
+    final String previousMode = GlobalConfiguration.BUCKET_REUSE_SPACE_MODE.getValueAsString();
+    GlobalConfiguration.BUCKET_REUSE_SPACE_MODE.setValue("high");
+
+    FileUtils.deleteRecursively(new File(DB_PATH));
+    try (final DatabaseFactory factory = new DatabaseFactory(DB_PATH)) {
+      // Step 1: create the database and fill the bucket with many pages. Default page size is
+      // ~64KB; 5_000 records with a 1 KB payload span dozens of pages, enough to make a full
+      // bucket-page scan during schema reload measurable.
+      try (final Database db = factory.create()) {
+        db.getSchema().createDocumentType(TYPE, 1);
+        final String payload = "x".repeat(PAYLOAD_KB * 1024);
+        db.transaction(() -> {
+          for (int i = 0; i < REC_COUNT; i++)
+            db.newDocument(TYPE).set("idx", i).set("payload", payload).save();
+        });
+      }
+
+      try (final Database db = factory.open()) {
+        final DatabaseInternal internal = (DatabaseInternal) db;
+        final PageManager pageManager = internal.getPageManager();
+
+        // The open path warmed schema state. Evict everything we cached for this database so the
+        // next reload has to go to disk - the same situation a Raft follower hits during a
+        // SCHEMA_ENTRY apply after a long bulk-load run that has churned the page cache.
+        pageManager.removeAllReadPagesOfDatabase(db);
+
+        final long pagesReadBaseline = pageManager.getStats().pagesRead;
+
+        // Simulate the follower SCHEMA_ENTRY path: full schema reload from disk.
+        ((LocalSchema) internal.getSchema().getEmbedded()).load(ComponentFile.MODE.READ_WRITE, true);
+
+        final long pagesReadDelta = pageManager.getStats().pagesRead - pagesReadBaseline;
+
+        // Before the fix, gatherPageStatistics() ran from the bucket constructor on every load
+        // and read every full page (often hundreds for a 5_000-record bucket). After the fix,
+        // schema reload touches only the dictionary, schema.json, and bucket headers - a small,
+        // fixed amount of I/O independent of bucket size.
+        assertThat(pagesReadDelta)
+            .as("Schema reload must not eagerly scan bucket data pages (issue #4219)")
+            .isLessThan(30L);
+      }
+    } finally {
+      GlobalConfiguration.BUCKET_REUSE_SPACE_MODE.setValue(previousMode);
+      FileUtils.deleteRecursively(new File(DB_PATH));
+    }
+  }
+
+  @Test
+  void lazyPageStatisticsStillWorkAfterReload() throws Exception {
+    // Sanity check that removing the eager warmup did not regress the lazy path: in HIGH mode,
+    // the first allocation after a reload must still trigger gatherPageStatistics() (so writes
+    // continue to recycle freed space on a leader). Measured indirectly via pagesRead delta.
+    final String previousMode = GlobalConfiguration.BUCKET_REUSE_SPACE_MODE.getValueAsString();
+    GlobalConfiguration.BUCKET_REUSE_SPACE_MODE.setValue("high");
+
+    FileUtils.deleteRecursively(new File(DB_PATH));
+    try (final DatabaseFactory factory = new DatabaseFactory(DB_PATH)) {
+      try (final Database db = factory.create()) {
+        db.getSchema().createDocumentType(TYPE, 1);
+        final String payload = "y".repeat(PAYLOAD_KB * 1024);
+        db.transaction(() -> {
+          for (int i = 0; i < REC_COUNT; i++)
+            db.newDocument(TYPE).set("idx", i).set("payload", payload).save();
+        });
+      }
+
+      try (final Database db = factory.open()) {
+        final DatabaseInternal internal = (DatabaseInternal) db;
+        final PageManager pageManager = internal.getPageManager();
+
+        // Force a fresh schema reload and drop cached pages so the lazy scan we want to trigger
+        // below has to read from disk (which is what bumps pagesRead).
+        ((LocalSchema) internal.getSchema().getEmbedded()).load(ComponentFile.MODE.READ_WRITE, true);
+        pageManager.removeAllReadPagesOfDatabase(db);
+
+        final long pagesReadBeforeAlloc = pageManager.getStats().pagesRead;
+
+        // First write after the reload: the allocator must run gatherPageStatistics lazily,
+        // which scans pages and increases the page-read counter (proving the lazy path fires).
+        db.transaction(() -> db.newDocument(TYPE).set("idx", -1).set("payload", "trigger").save());
+
+        final long pagesReadAfterAlloc = pageManager.getStats().pagesRead;
+        assertThat(pagesReadAfterAlloc - pagesReadBeforeAlloc)
+            .as("First allocation after a reload must trigger lazy gatherPageStatistics")
+            .isGreaterThan(0L);
+      }
+    } finally {
+      GlobalConfiguration.BUCKET_REUSE_SPACE_MODE.setValue(previousMode);
+      FileUtils.deleteRecursively(new File(DB_PATH));
+    }
+  }
+}

ha-raft/src/main/java/com/arcadedb/server/ha/raft/ArcadeStateMachine.java

@@ -126,6 +126,14 @@ public record BootstrapBaseline(String fingerprint, long lastTxId) {
 
   private final AtomicBoolean needsSnapshotDownload = new AtomicBoolean(false);
   private final AtomicBoolean catchingUp            = new AtomicBoolean(false);
+  // Set to true after applyTransaction catches an unexpected Throwable (OOM, NPE, etc.). The
+  // state machine's in-memory schema/page state can be inconsistent at that point (issue #4219:
+  // mid-load OOM leaves bucketMap cleared but not repopulated), so any subsequent apply
+  // attempt would surface as a cascade of "Bucket with id X was not found" errors before the
+  // async server.stop() completes. Once tripped, applyTransaction fails fast without touching
+  // database state and the recovery path is the asynchronous server shutdown plus a snapshot
+  // resync on the next start.
+  private final AtomicBoolean haltedAfterCriticalError = new AtomicBoolean(false);
 
 
   public void setServer(final ArcadeDBServer server) {
@@ -235,6 +243,13 @@ public CompletableFuture<Message> applyTransaction(final TransactionContext trx)
     final TermIndex termIndex = TermIndex.valueOf(entry);
     final long index = termIndex.getIndex();
 
+    // Refuse to apply once a prior entry tripped the critical-error halt. Continuing would
+    // operate on the inconsistent in-memory state left behind by the failed apply and cascade
+    // into additional SEVERE errors before the async server.stop() completes (#4219).
+    if (haltedAfterCriticalError.get())
+      return CompletableFuture.failedFuture(new ReplicationException(
+          "State machine halted after critical error at earlier index; refusing to apply index " + index));
+
     try {
       final RaftLogEntryCodec.DecodedEntry decoded = RaftLogEntryCodec.decode(data);
 
@@ -299,6 +314,9 @@ public CompletableFuture<Message> applyTransaction(final TransactionContext trx)
           """
           CRITICAL: Unexpected error applying Raft log entry at index %d. \
           Shutting down to prevent state divergence.""", e, index);
+      // Trip the halt flag BEFORE starting the async server.stop() so the StateMachineUpdater's
+      // next applyTransaction call short-circuits instead of cascading on inconsistent state.
+      haltedAfterCriticalError.set(true);
       final Thread stopThread = new Thread(() -> {
         try {
           if (server != null)